Merge branch 'dev' of zwg/qc-dgp into master

README.md

@@ -23,8 +23,8 @@ Data grabbing platform(DGP)数据采集平台
 #### 参与贡献
 
 * Fork 本仓库，拉取的是dev分支
-* 直接在dev分支上开发,添加新的依赖后请执行 `pip freeze > requirements.txt` 或 `pipreqs /qc-dgp`
-* 提交代码
+* 直接在dev分支上开发,添加新的依赖后请执行 `pip freeze > requirements.txt` 或 `pipreqs --force /qc-dgp`
+* 格式化代码，提交代码
 * 新建 Pull Request
 
 

dgp/get_order_daily.py

@@ -3,7 +3,7 @@
 """
 Created on Mon May 18 13:13:00 2020
 @author: chencong
-## amend1  20200925 11:39:00
+## amend1  20200927 11:39:00
 ## amend2  
 
 """
@@ -545,15 +545,22 @@ def mysql_insert_order(data):
     cursor = db.cursor()   
     #sql = 'insert ignore into quchen_text.order_daily (amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
     #sql = "update quchen_text.order set amount =%s where platform='掌中云' and order_id =%s"
-    sql = 'replace into quchen_text.order_daily (amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
-
+    sql1 = 'replace into quchen_text.order_daily (amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
+    sql2 = 'replace into quchen_text.order(amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
+    try:
+        cursor.executemany(sql1,data)
+        db.commit()
+        print('access insert order_daily',len(data))
+    except:
+        db.rollback()
+        print('defeat order_daily')
     try:
-        cursor.executemany(sql,data)
+        cursor.executemany(sql2,data)
         db.commit()
         print('access insert order',len(data))
     except:
         db.rollback()
-        print('defeat')
+        print('defeat order')
 
 
 
@@ -561,8 +568,8 @@ def start_all_job():
     request_time_stamp = time.time()
     st_unix = int((request_time_stamp+8*3600)//86400*86400-8*3600-86400) 
     et_unix = int((request_time_stamp+8*3600)//86400*86400-8*3600)
-    st_dt = time.strftime("%Y-%m-%dT%H:%M:%S",time.localtime(st_unix))+'+08:00'
-    et_dt = time.strftime("%Y-%m-%dT%H:%M:%S",time.localtime(et_unix))+'+08:00'
+    st_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(st_unix)) + '+08:00'
+    et_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(et_unix)) + '+08:00'
     get_yuewen_order(st_unix,et_unix)
     get_ysg_order(st_unix,et_unix)
     get_zhangdu_order(st_unix,et_unix)  
@@ -571,7 +578,7 @@ def start_all_job():
 
 
 
-start_job_time = '2020-09-26 03:05:00'
+start_job_time = '2020-09-27 03:05:00'
 
 
 if __name__ == '__main__':
@@ -579,4 +586,3 @@ if __name__ == '__main__':
     scheduler.add_job(start_all_job, 'interval',days=1,start_date=start_job_time)
     scheduler.start()
 
-

dgp/tests/LoggerService.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+__title__ = '日志格式化器'
+
+@Time    : 2020/9/25 13:36
+@Author  : zhengwangeng
+@Software: PyCharm
+
+# code is far away from bugs with the god animal protecting
+    I love animals. They taste delicious.
+              ┏┓      ┏┓
+            ┏┛┻━━━┛┻┓
+            ┃      ☃      ┃
+            ┃  ┳┛  ┗┳  ┃
+            ┃      ┻      ┃
+            ┗━┓      ┏━┛
+                ┃      ┗━━━┓
+                ┃  神兽保佑    ┣┓
+                ┃　永无BUG！   ┏┛
+                ┗┓┓┏━┳┓┏┛
+                  ┃┫┫  ┃┫┫
+                  ┗┻┛  ┗┻┛
+"""
+
+import logging
+import sys
+
+from logging.handlers import TimedRotatingFileHandler
+
+
+class LoggerService:
+
+    @staticmethod
+    def logger_timefile(log_file, log_name, backupCount=10):
+        logger = logging.getLogger(log_name)
+        logger.setLevel(logging.DEBUG)
+        # format
+        formatter = logging.Formatter(fmt='%(asctime)s - %(filename)s[%(lineno)d] - %(levelname)s - %(message)s',
+                                      datefmt='%Y/%m/%d %H:%M:%S')
+
+        # StreamHandler
+        stream_handler = logging.StreamHandler(sys.stdout)
+        stream_handler.setFormatter(formatter)
+        logger.addHandler(stream_handler)
+
+        # 创建TimedRotatingFileHandler对象
+        file_handler2 = TimedRotatingFileHandler(filename=log_file, when="D", interval=1, backupCount=backupCount)
+        file_handler2.setFormatter(formatter)
+        logger.addHandler(file_handler2)
+
+        return logger
+
+    @staticmethod
+    def logger_file(log_file, log_name):
+        logger = logging.getLogger(log_name)
+        logger.setLevel(logging.DEBUG)
+        # format
+        formatter = logging.Formatter(fmt='%(asctime)s - %(filename)s[%(lineno)d] - %(levelname)s - %(message)s',
+                                      datefmt='%Y/%m/%d %H:%M:%S')
+
+        # StreamHandler
+        stream_handler = logging.StreamHandler(sys.stdout)
+        stream_handler.setFormatter(formatter)
+        logger.addHandler(stream_handler)
+
+        # # FileHandler
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+
+        return logger

dgp/tests/MySQLConnection.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+__title__ = 'MySQL连接池对象'
+
+@Time    : 2020/9/26 18:36
+@Author  : Kenny-PC
+@Software: PyCharm
+
+# code is far away from bugs with the god animal protecting
+    I love animals. They taste delicious.
+              ┏┓      ┏┓
+            ┏┛┻━━━┛┻┓
+            ┃      ☃      ┃
+            ┃  ┳┛  ┗┳  ┃
+            ┃      ┻      ┃
+            ┗━┓      ┏━┛
+                ┃      ┗━━━┓
+                ┃  神兽保佑    ┣┓
+                ┃　永无BUG！   ┏┛
+                ┗┓┓┏━┳┓┏┛
+                  ┃┫┫  ┃┫┫
+                  ┗┻┛  ┗┻┛
+"""
+"""
+数据库连接池相关
+"""
+
+import configparser
+import logging
+import os
+
+import pymysql
+from DBUtils.PooledDB import PooledDB
+
+# 获取当前文件所在目录的上一级目录
+root_dir = os.path.dirname(os.path.abspath('.'))
+print('当前项目根目录为：', root_dir)
+
+# 读取数据库配置信息
+config = configparser.ConfigParser()
+config.read(root_dir + '/tests/conf/db.ini', encoding='UTF-8')
+sections = config.sections()
+
+# 数据库工厂
+dbFactory = {}
+for dbName in sections:
+    # 读取相关属性
+    maxconnections = config.get(dbName, "maxconnections")
+    mincached = config.get(dbName, "mincached")
+    maxcached = config.get(dbName, "maxcached")
+    host = config.get(dbName, "host")
+    port = config.get(dbName, "port")
+    user = config.get(dbName, "user")
+    password = config.get(dbName, "password")
+    database = config.get(dbName, "database")
+    databasePooled = PooledDB(creator=pymysql,
+                              maxconnections=int(maxconnections),
+                              mincached=int(mincached),
+                              maxcached=int(maxcached),
+                              blocking=True,
+                              cursorclass=pymysql.cursors.DictCursor,
+                              host=host,
+                              port=int(port),
+                              user=user,
+                              password=password,
+                              database=database)
+    dbFactory[dbName] = databasePooled
+
+
+class MySQLConnection(object):
+    """
+    数据库连接池代理对象
+    查询参数主要有两种类型
+    第一种：传入元祖类型,例如(12,13),这种方式主要是替代SQL语句中的%s展位符号
+    第二种: 传入字典类型,例如{"id":13},此时我们的SQL语句需要使用键来代替展位符,例如：%(name)s
+    """
+
+    def __init__(self, dbName="test"):
+        self.connect = dbFactory[dbName].connection()
+        self.cursor = self.connect.cursor()
+        logging.debug("获取数据库连接对象成功,连接池对象:{}".format(str(self.connect)))
+
+    def execute(self, sql, param=None):
+        """
+        基础更新、插入、删除操作
+        :param sql:
+        :param param:
+        :return: 受影响的行数
+        """
+        ret = None
+        try:
+            if param == None:
+                ret = self.cursor.execute(sql)
+            else:
+                ret = self.cursor.execute(sql, param)
+        except TypeError as te:
+            logging.debug("类型错误")
+            logging.exception(te)
+        return ret
+
+    def query(self, sql, param=None):
+        """
+        查询数据库
+        :param sql: 查询SQL语句
+        :param param: 参数
+        :return: 返回集合
+        """
+        self.cursor.execute(sql, param)
+        result = self.cursor.fetchall()
+        return result
+
+    def queryOne(self, sql, param=None):
+        """
+        查询数据返回第一条
+        :param sql: 查询SQL语句
+        :param param: 参数
+        :return: 返回第一条数据的字典
+        """
+        result = self.query(sql, param)
+        if result:
+            return result[0]
+        else:
+            return None
+
+    def listByPage(self, sql, current_page, page_size, param=None):
+        """
+        分页查询当前表格数据
+        :param sql: 查询SQL语句
+        :param current_page: 当前页码
+        :param page_size: 页码大小
+        :param param:参数
+        :return:
+        """
+        countSQL = "select count(*) ct from (" + sql + ") tmp "
+        logging.debug("统计SQL:{}".format(sql))
+        countNum = self.count(countSQL, param)
+        offset = (current_page - 1) * page_size
+        totalPage = int(countNum / page_size)
+        if countNum % page_size > 0:
+            totalPage = totalPage + 1
+        pagination = {"current_page": current_page, "page_size": page_size, "count": countNum, "total_page": totalPage}
+        querySql = "select * from (" + sql + ") tmp limit %s,%s"
+        logging.debug("查询SQL:{}".format(querySql))
+        # 判断是否有参数
+        if param == None:
+            # 无参数
+            pagination["data"] = self.query(querySql, (offset, page_size))
+        else:
+            # 有参数的情况,此时需要判断参数是元祖还是字典
+            if isinstance(param, dict):
+                # 字典的情况,因此需要添加字典
+                querySql = "select * from (" + sql + ") tmp limit %(tmp_offset)s,%(tmp_pageSize)s"
+                param["tmp_offset"] = offset
+                param["tmp_pageSize"] = page_size
+                pagination["data"] = self.query(querySql, param)
+            elif isinstance(param, tuple):
+                # 元祖的方式
+                listtp = list(param)
+                listtp.append(offset)
+                listtp.append(page_size)
+                pagination["data"] = self.query(querySql, tuple(listtp))
+            else:
+                # 基础类型
+                listtp = []
+                listtp.append(param)
+                listtp.append(offset)
+                listtp.append(page_size)
+                pagination["data"] = self.query(querySql, tuple(listtp))
+        return pagination
+
+    def count(self, sql, param=None):
+        """
+        统计当前表记录行数
+        :param sql: 统计SQL语句
+        :param param: 参数
+        :return: 当前记录行
+        """
+        ret = self.queryOne(sql, param)
+        count = None
+        if ret:
+            for k, v in ret.items():
+                count = v
+        return count
+
+    def insert(self, sql, param=None):
+        """
+        数据库插入
+        :param sql: SQL语句
+        :param param: 参数
+        :return: 受影响的行数
+        """
+        return self.execute(sql, param)
+
+    def update(self, sql, param=None):
+        """
+        更新操作
+        :param sql: SQL语句
+        :param param: 参数
+        :return: 受影响的行数
+        """
+        return self.execute(sql, param)
+
+    def delete(self, sql, param=None):
+        """
+        删除操作
+        :param sql: 删除SQL语句
+        :param param: 参数
+        :return: 受影响的行数
+        """
+        return self.execute(sql, param)
+
+    def batch(self, sql, param=None):
+        """
+        批量插入
+        :param sql: 插入SQL语句
+        :param param: 参数
+        :return: 受影响的行数
+        """
+        return self.cursor.executemany(sql, param)
+
+    def commit(self, param=None):
+        """
+        提交数据库
+        :param param:
+        :return:
+        """
+        if param == None:
+            self.connect.commit()
+        else:
+            self.connect.rollback()
+
+    def close(self):
+        """
+        关闭数据库连接
+        :return:
+        """
+        if self.cursor:
+            self.cursor.close()
+        if self.connect:
+            self.connect.close()
+        logging.debug("释放数据库连接")
+        return None

dgp/tests/__init__.py → dgp/tests/QueryType.py

@@ -1,10 +1,13 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+
 """
-__title__ = 'Package_name' __author__ = '
-Package_name' __author__ = '
-USER'
-__mtime__ = '2020/9/24'
+__title__ = '查询类型，按小时查还是按天查'
+
+@Time    : 2020/9/27 13:41
+@Author  : zhengwangeng
+@Software: PyCharm
+
 # code is far away from bugs with the god animal protecting
     I love animals. They taste delicious.
               ┏┓      ┏┓
@@ -21,3 +24,10 @@ __mtime__ = '2020/9/24'
                   ┗┻┛  ┗┻┛
 """
 
+from enum import Enum, unique
+
+
+@unique
+class QueryType(Enum):
+    HOURE = 1
+    DAY = 2

dgp/tests/check_order.py

@@ -0,0 +1,623 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+__title__ = '每日凌晨空闲时检查本地数据库中的订单数据是否和平台昨天总订单一致'
+
+@Time    : 2020/9/26 19:44
+@Author  : Kenny-PC
+@Software: PyCharm
+
+# code is far away from bugs with the god animal protecting
+    I love animals. They taste delicious.
+              ┏┓      ┏┓
+            ┏┛┻━━━┛┻┓
+            ┃      ☃      ┃
+            ┃  ┳┛  ┗┳  ┃
+            ┃      ┻      ┃
+            ┗━┓      ┏━┛
+                ┃      ┗━━━┓
+                ┃  神兽保佑    ┣┓
+                ┃　永无BUG！   ┏┛
+                ┗┓┓┏━┳┓┏┛
+                  ┃┫┫  ┃┫┫
+                  ┗┻┛  ┗┻┛
+"""
+
+import datetime
+import hashlib
+import math
+import time
+from urllib import parse
+
+import requests
+
+import account_list as al
+from util import date_util, platform_util
+
+
+def md5value(s):
+    md5 = hashlib.md5()
+    md5.update(s.encode("utf-8"))
+    return md5.hexdigest()
+
+
+##《1》阅文
+def get_yuewen_order(st, et):
+    url = 'https://open.yuewen.com/cpapi/wxRecharge/querychargelog'
+    version = 1
+    secert_list = al.yuewen_account_list
+
+    for secert in secert_list:
+        start_time = st
+        email = secert[0]
+        appsecert = secert[1]
+
+        for i in range(int((et - st) / 86400)):
+            t = ()
+            end_time = min(start_time + 86400, et)
+            timestamp = int(time.time())
+            s = ''
+            page = 1
+            order_status = 2
+            data = {
+                'email': email,
+                'version': version,
+                'timestamp': timestamp,
+                'start_time': start_time,
+                'end_time': end_time,
+                'page': page,
+                'order_status': order_status
+                # 'last_min_id':last_min_id,
+                # 'last_max_id':last_max_id,
+                # 'total_count':total_count,
+                # 'last_page':last_page
+            }
+            sorted_data = sorted(data.items())
+            for k, v in sorted_data:
+                s = s + str(k) + str(v)
+
+            sign = md5value(appsecert + s).upper()
+
+            data1 = {
+                'email': email,
+                'version': version,
+                'timestamp': timestamp,
+                'start_time': start_time,
+                'end_time': end_time,
+                'page': page,
+                'order_status': order_status,
+                'sign': sign
+            }
+            list1 = requests.get(url=url, params=data1)
+
+            total_count = list1.json()['data']['total_count']
+            last_min_id = list1.json()['data']['min_id']
+            last_max_id = list1.json()['data']['max_id']
+            last_page = list1.json()['data']['page']
+
+            if total_count > 0:
+                for x in list1.json()['data']['list']:
+                    y = {}
+                    dtime = datetime.datetime.strptime(x['order_time'], "%Y-%m-%d %H:%M:%S")
+                    y['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
+                    y['platform'] = '阅文'
+                    y['channel'] = x['app_name']
+                    y['from_novel'] = x['book_name']
+                    y['user_id'] = x['openid']
+                    y['stage'] = ''
+                    y['channel_id'] = 0
+                    y['order_time'] = x['order_time']
+                    y['amount'] = x['amount']
+                    y['reg_time'] = x['reg_time']
+                    y['order_id'] = x['order_id']
+                    """
+                    del x['app_name']
+                    del x['order_status']
+                    del x['order_type']
+                    del x['openid']
+                    del x['user_name']
+                    del x['sex']
+                    del x['channel_name']
+                    del x['book_id']
+                    del x['book_name']
+                    del x['report_status']
+                    """
+                    y = sorted(y.items(), key=lambda item: item[0])
+                    y = dict(y)
+                    y = tuple(y.values())
+                    t = t + ((y),)
+
+            if total_count > 100:
+                for page in range(2, math.ceil(total_count / 100) + 1):
+                    data = {
+                        'email': email,
+                        'version': version,
+                        'timestamp': timestamp,
+                        'start_time': start_time,
+                        'end_time': end_time,
+                        'page': page,
+                        'last_min_id': last_min_id,
+                        'last_max_id': last_max_id,
+                        'total_count': total_count,
+                        'last_page': last_page,
+                        'order_status': order_status
+                    }
+                    sorted_data = sorted(data.items())
+                    s1 = ''
+                    for k, v in sorted_data:
+                        s1 = s1 + str(k) + str(v)
+                        sign = md5value(appsecert + s1).upper()
+                        data2 = {
+                            'email': email,
+                            'version': version,
+                            'timestamp': timestamp,
+                            'start_time': start_time,
+                            'end_time': end_time,
+                            'page': page,
+                            'last_min_id': last_min_id,
+                            'last_max_id': last_max_id,
+                            'total_count': total_count,
+                            'last_page': last_page,
+                            'order_status': order_status,
+                            'sign': sign
+                        }
+                    list2 = requests.get(url=url, params=data2)
+                    for x in list2.json()['data']['list']:
+                        y = {}
+                        dtime = datetime.datetime.strptime(x['order_time'], "%Y-%m-%d %H:%M:%S")
+                        y['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
+                        y['platform'] = '阅文'
+                        y['channel'] = x['app_name']
+                        y['from_novel'] = x['book_name']
+                        y['user_id'] = x['openid']
+                        y['stage'] = ''
+                        y['channel_id'] = 0
+                        y['order_time'] = x['order_time']
+                        y['amount'] = x['amount']
+                        y['reg_time'] = x['reg_time']
+                        y['order_id'] = x['order_id']
+                        """
+                        del x['app_name']
+                        del x['order_status']
+                        del x['order_type']
+                        del x['openid']
+                        del x['user_name']
+                        del x['sex']
+                        del x['channel_name']
+                        del x['book_id']
+                        del x['book_name']
+                        del x['report_status']
+                        """
+                        y = sorted(y.items(), key=lambda item: item[0])
+                        y = dict(y)
+                        y = tuple(y.values())
+                        t = t + ((y),)
+
+                    total_count = list2.json()['data']['total_count']
+                    last_min_id = list2.json()['data']['min_id']
+                    last_max_id = list2.json()['data']['max_id']
+                    last_page = list2.json()['data']['page']
+
+            print(email, start_time, len(t))
+            start_time = start_time + 86400
+            if len(t) > 0:
+                mysql_insert_order(t)
+
+
+##《2》掌读
+def get_zhangdu_order(st, et):
+    secert_list = al.zhangdu_account_list
+    url = 'https://api.zhangdu520.com/channel/getorder'
+
+    for item in secert_list:  # 分渠道
+        t = ()
+
+        uid = item[0]
+        appsecert = item[1]
+        channel = item[2]
+        timestamp = int(time.time())
+        sign = md5value(str(uid) + '&' + appsecert + '&' + str(timestamp))
+        page = 1
+        starttime = st
+        timespace = 90 * 3600 * 24
+        endtime = min(et, st + timespace)
+
+        for x in range((et - st) // timespace + 1):  # 分时段
+            Params = {
+                'uid': uid,
+                'timestamp': timestamp,
+                'sign': sign,
+                'starttime': starttime,
+                'endtime': endtime
+            }
+            list1 = requests.get(url=url, params=Params)
+            pageCount = list1.json()['data']['pageCount']
+            if pageCount > 0:
+                for a in range(1, pageCount + 1):  # 分页
+                    page = a
+                    Params = {
+                        'uid': uid,
+                        'timestamp': timestamp,
+                        'sign': sign,
+                        'starttime': starttime,
+                        'endtime': endtime,
+                        'page': page
+                    }
+                    list2 = requests.get(url=url, params=Params).json()
+                    if 'data' in list2.keys():
+                        for b in list2['data']['list']:
+                            c = {}
+                            c['amount'] = b['amount']
+                            c['channel_id'] = uid
+                            c['order_id'] = str(b['orderno'])
+                            c['order_time'] = b['ctime']
+                            c['user_id'] = b['openid']
+                            c['platform'] = '掌读'
+                            c['channel'] = channel
+                            c['reg_time'] = b['regtime']
+                            c['from_novel'] = ''
+                            c['stage'] = ''
+                            c['date'] = ((int(b['ctime']) + 8 * 3600) // 86400) * 86400 - 8 * 3600
+                            """
+                            del b['openid']
+                            del b['regtime']
+                            del b['ip']
+                            del b['ua']
+                            del b['id']
+                            del b['ctime']
+                            del b['userid']
+                            del b['orderno']
+                            del b['source']
+                            del b['sourceid']
+                            """
+                            if b['status'] == '1':
+                                # del b['status']
+                                del b
+                                x = sorted(c.items(), key=lambda item: item[0])
+                                x = dict(x)
+                                x = tuple(x.values())
+                                t = t + ((x),)
+                    else:
+                        print(list2)
+            starttime = starttime + timespace
+            endtime = min(et, starttime + timespace)
+        if len(t) > 0:
+            mysql_insert_order(t)
+        print('掌读', channel, len(t))
+
+
+##《3》花生
+def get_huasheng_order(st, et):
+    apikey_list = al.huasheng_account_list
+    url = 'https://vip.rlcps.cn/api/getMerchants'
+
+    for key in apikey_list:  # 获取每个vip账号下的channel_id
+        apiKEY = key[0]
+        apiSecurity = key[1]
+        stage = key[2]
+        timestamp = str(int(time.time()))
+        sign = md5value(apiKEY + timestamp + apiSecurity).upper()
+        data = {
+            'apiKey': apiKEY,
+            'apiSecurity': apiSecurity,
+            'timestamp': timestamp,
+            'sign': sign
+        }
+        list0 = requests.post(url, data).json()
+        t = ()
+
+        for merchant in list0['data']:
+            merchant_id = merchant['merchant_id']
+            merchant_name = merchant['merchant_name']
+            url1 = 'https://vip.rlcps.cn/api/orderList'
+            start_time = st
+
+            for i in range((et - st) // 86400):
+                page = 1
+                date = time.strftime("%Y-%m-%d", time.localtime(start_time))
+                sign = md5value(apiKEY + date + str(merchant_id) + timestamp + apiSecurity).upper()
+                data1 = {
+                    'apiKey': apiKEY,
+                    'apiSecurity': apiSecurity,
+                    'timestamp': timestamp,
+                    'date': date,
+                    'merchant_id': merchant_id,
+                    'sign': sign,
+                    'page': page
+                }
+                list1 = requests.post(url1, data1).json()
+
+                if 'data' in list1.keys() and len(list1['data']) > 0:
+
+                    for i in range(int(math.ceil(list1['count'] / 500))):
+                        data2 = {
+                            'apiKey': apiKEY,
+                            'apiSecurity': apiSecurity,
+                            'timestamp': timestamp,
+                            'date': date,
+                            'merchant_id': merchant_id,
+                            'sign': sign,
+                            'page': page
+                        }
+                        list2 = requests.post(url1, data2).json()
+
+                        for x in list2['data']:
+                            if x['order_status'] == 1:
+                                y = {}
+                                ##dtime = datetime.datetime.strptime(x['pay_at'],"%Y-%m-%d")
+                                ##y['date']= ((int(time.mktime(dtime.timetuple()))+8*3600)//86400)*86400-8*3600
+                                y['user_id'] = x['openid']
+                                y['order_id'] = x['trans_id']
+                                y['order_time'] = x['pay_at']
+                                y['reg_time'] = x['join_at']
+                                y['date'] = (start_time + 8 * 3600) // 86400 * 86400 - 8 * 3600
+                                y['channel'] = merchant_name
+                                y['channel_id'] = merchant_id
+                                y['platform'] = '花生'
+                                y['stage'] = stage
+                                y['from_novel'] = x['book_name']
+                                y['amount'] = x['amount']
+                                """
+                                del x['order_num']
+                                del x['book_name']
+                                del x['trans_id']
+                                del x['pay_at']
+                                del x['join_at']
+                                del x['subscribe_at']
+                                del x['openid']
+                                del x['charge_count']
+                                del x['book_id']
+                                del x['order_status']
+                                del x['user_name']
+                                del x['spread_name']
+                                del x['request_at']
+                                """
+                                y = sorted(y.items(), key=lambda item: item[0])
+                                y = dict(y)
+                                y = tuple(y.values())
+                                t = t + ((y),)
+                        page = page + 1
+                else:
+                    print(list1)
+                start_time = start_time + 86400
+
+        if len(t) > 0:
+            mysql_insert_order(t)
+    print(stage, merchant_name, len(t))
+
+
+##《4》掌中云
+def get_zzy_order(st, et):
+    # 掌中云的时间格式比较特殊,转换下
+    st = platform_util.getZzyQueryTime(st)
+    et = platform_util.getZzyQueryTime(et)
+
+    API_list = al.zzy_account_list
+    url = 'https://openapi.818tu.com/partners/channel/channels/list?'
+
+    for x in API_list:
+        my_key = x[0]
+        secert = x[1]
+        stage = x[2]
+        my_sign = md5value(secert + 'key=' + my_key)
+        parameter = 'key=' + my_key + '&sign=' + my_sign
+        channel_list = requests.get(url + parameter)  # 获取子渠道列表
+
+        if 'data' in channel_list.json().keys():
+            items = channel_list.json()['data']['items']
+        elif len(x) > 3:
+            my_key = x[3]
+            secert = x[4]
+            my_sign = md5value(secert + 'key=' + my_key)
+            parameter = 'key=' + my_key + '&sign=' + my_sign
+            channel_list = requests.get(url + parameter)
+            items = channel_list.json()['data']['items']
+        else:
+            print(channel_list.json())
+            items = []
+
+        for item in items:  # 获取channel_id 后逐个拉取历史orders
+            r = ()
+            channel_id = item['id']
+            channel = item['nickname']
+            status = str(1)
+            per_page = str(1000)
+            limit_time = et
+            get_time = st
+            lt = parse.urlencode({'created_at[lt]': limit_time})
+            gt = parse.urlencode({'created_at[gt]': get_time})
+            url_1 = 'https://openapi.818tu.com/partners/channel/orders/list?'
+            my_sign_1 = md5value(secert + 'channel_id=' + str(
+                channel_id) + '&created_at[gt]=' + get_time + '&created_at[lt]=' + limit_time + '&key=' + my_key + '&per_page=' + per_page + '&status=' + status)
+            parameter_1 = 'channel_id=' + str(
+                channel_id) + '&' + gt + '&' + lt + '&per_page=' + per_page + '&status=' + status + '&key=' + my_key + '&sign=' + my_sign_1
+            orders = requests.get(url_1 + parameter_1)
+            t = orders.json()['data']['count'] // int(per_page) + 1
+            for page in range(1, t + 1):
+                my_sign_2 = md5value(secert + 'channel_id=' + str(
+                    channel_id) + '&created_at[gt]=' + get_time + '&created_at[lt]=' + limit_time + '&key=' + my_key + '&page=' + str(
+                    page) + '&per_page=' + per_page + '&status=' + status)
+                parameter_2 = 'channel_id=' + str(channel_id) + '&' + gt + '&' + lt + '&page=' + str(
+                    page) + '&per_page=' + per_page + '&status=' + status + '&key=' + my_key + '&sign=' + my_sign_2
+                orders_1 = requests.get(url_1 + parameter_2)
+                b = orders_1.json()['data']['items']
+
+                for a in b:
+                    c = {}
+                    c['user_id'] = str(a['member']['openid'])
+                    c['channel'] = channel
+                    c['reg_time'] = a['member']['created_at']
+                    c['channel_id'] = channel_id
+                    c['amount'] = round(a['price'] / 100, 2)
+                    c['order_id'] = str(a['id'])
+                    c['order_time'] = a['created_at']
+                    c['platform'] = '掌中云'
+                    c['stage'] = stage
+                    # c['amount']=a['amount']
+                    dtime = datetime.datetime.strptime(a['created_at'][0:10], "%Y-%m-%d")
+                    c['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
+
+                    if str(a['from_novel_id']) != 'None':
+                        c['from_novel'] = a['from_novel']['title']
+                    else:
+                        c['from_novel'] = 'None'
+                    """
+                    del a['member']
+                    del a['referral_link_id']
+                    del a['id']
+                    del a['created_at']
+                    del a['paid_at']
+                    del a['border_id']
+                    del a['from_novel_id']
+                    del a['status']
+                    del a['price']
+                    del a['agent_uid']
+                    """
+                    x = sorted(c.items(), key=lambda item: item[0])
+                    x = dict(x)
+                    x = tuple(x.values())
+                    r = r + ((x),)
+
+            if len(r) > 0:
+                mysql_insert_order(r)
+
+            print('zzy', channel, len(r))
+
+
+##《5》 悠书阁
+def get_ysg_order(st, et):
+    key_list = al.ysg_account_list
+    url = 'https://novel.youshuge.com/v2/open/orders'
+    o = ()
+
+    for key in key_list:
+        host_name = key[0]
+        channel_id = key[1]
+        secert_key = key[2]
+        channel = key[3]
+        stage = key[4]
+        timestamp = int(time.time())
+        start_date = time.strftime("%Y-%m-%d", time.localtime(st))
+        end_date = time.strftime("%Y-%m-%d", time.localtime(et))
+        page = 1
+        str1 = 'channel_id=' + str(channel_id) + '&end_date=' + end_date + '&host_name=' + host_name + '&page=' + str(
+            page) + '&pay_status=1' + '&start_date=' + start_date + '&time=' + str(timestamp) + '&key=' + secert_key
+        sign = md5value(str1).upper()
+        data = {
+            'sign': sign,
+            'host_name': host_name,
+            'time': timestamp,
+            'channel_id': channel_id,
+            'page': page,
+            'pay_status': 1,
+            'start_date': start_date,
+            'end_date': end_date
+        }
+        r = requests.post(url, data).json()
+
+        if 'data' in r.keys():
+            if len(r['data']) > 0:
+                for i in range((r['data'][0]['count'] - 1) // 100 + 1):
+                    timestamp = int(time.time())
+                    str1 = 'channel_id=' + str(
+                        channel_id) + '&end_date=' + end_date + '&host_name=' + host_name + '&page=' + str(
+                        page) + '&pay_status=1' + '&start_date=' + start_date + '&time=' + str(
+                        timestamp) + '&key=' + secert_key
+                    sign = md5value(str1).upper()
+                    data2 = {
+                        'sign': sign,
+                        'host_name': host_name,
+                        'time': timestamp,
+                        'channel_id': channel_id,
+                        'page': page,
+                        'pay_status': 1,
+                        'start_date': start_date,
+                        'end_date': end_date
+                    }
+                    r2 = requests.post(url, data2).json()
+
+                    if 'data' in r2.keys():
+                        if len(r2['data']) > 0:
+                            for x in r2['data']:
+                                y = {}
+                                dtime = datetime.datetime.strptime(x['create_time'][0:10], "%Y-%m-%d")
+                                y['date'] = ((int(
+                                    time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
+                                y['order_id'] = x['order_num']
+                                y['amount'] = round(int(x['price']) / 100, 2)
+                                y['order_time'] = x['create_time']
+                                y['channel'] = channel
+                                y['from_novel'] = x['book_name']
+                                y['stage'] = stage
+                                y['user_id'] = x['openid']
+                                y['channel_id'] = channel_id
+                                y['platform'] = '悠书阁'
+                                y['reg_time'] = x['reg_time']
+
+                                y = sorted(y.items(), key=lambda item: item[0])
+                                y = dict(y)
+                                y = tuple(y.values())
+                                o = o + ((y),)
+                    page = page + 1
+    if len(o) > 0:
+        mysql_insert_order(o)
+
+
+## 数据导入表采用replace替换主键orderid的方法
+def mysql_insert_order(data):
+    pass
+
+
+# if len(data) != 0:
+# 	print('数据为空，不执行数据库操作！')
+# 	pass
+# else:
+# 	db = pymysql.connect('localhost', 'root', 'root', 'quchen_text')
+# 	cursor = db.cursor()
+# 	# sql = 'insert ignore into quchen_text.order_daily (amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
+# 	# sql = "update quchen_text.order set amount =%s where platform='掌中云' and order_id =%s"
+# 	sql = 'replace into quchen_text.order (amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
+#
+# 	try:
+# 		cursor.executemany(sql, data)
+# 		db.commit()
+# 		print('access insert order', len(data))
+# 	except Exception as e:
+# 		db.rollback()
+# 		print('订单数据入库失败：', e)
+
+
+def start_all_job():
+    st_unix = date_util.getYesterdayStartTime()
+    et_unix = date_util.getTodayStartTime()
+
+    print(date_util.getSecondsToDatetime(st_unix))
+    print(date_util.getSecondsToDatetime(et_unix))
+
+    start_exec_seconds = date_util.getCurrentSecondTime()
+    # get_yuewen_order(st_unix, et_unix)
+    # get_ysg_order(st_unix, et_unix)
+    # get_zhangdu_order(st_unix, et_unix)
+
+    get_zzy_order(st_unix, et_unix)
+    # get_huasheng_order(st_unix, et_unix)
+    print('执行时间（秒）为：', date_util.getCurrentSecondTime() - start_exec_seconds)
+
+
+if __name__ == '__main__':
+    start_all_job()
+
+#
+# scheduler = BlockingScheduler()
+#
+# #每天凌晨执行
+# #start_job_time = '2020-09-26 03:05:00'
+# # scheduler.add_job(start_all_job, 'interval', days=1, start_date=start_job_time)
+#
+# #每天凌晨3点到4点的30分钟都执行一次
+# # scheduler.add_job(start_all_job, 'cron', hour='3-4', minute='30')
+#
+# # 每10秒执行一次
+# scheduler.add_job(start_all_job, 'interval', seconds=2)
+# scheduler.start()

dgp/tests/conf/db.ini

@@ -0,0 +1,35 @@
+;[pro]
+;# 数据库连接主机
+;host=rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com
+;# 数据库端口号
+;port=3306
+;# 用户名
+;user=superc
+;# 密码
+;password=Cc719199895
+;# 数据库名称
+;database=quchen_text
+;# 数据库连接池最大连接数
+;maxconnections=20
+;# 数据库连接池最小缓存数
+;mincached=5
+;# 数据库连接池最大缓存数
+;maxcached=10
+
+[test]
+# 数据库连接主机
+host = localhost
+# 数据库端口号
+port = 3306
+# 用户名
+user = root
+# 密码
+password = root
+# 数据库名称
+database = quchen_text
+# 数据库连接池最大连接数
+maxconnections = 20
+# 数据库连接池最小缓存数
+mincached = 5
+# 数据库连接池最大缓存数
+maxcached = 10

dgp/tests/test.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+__title__ = '测试类'
+
+@Time    : 2020/9/24 19:44
+@Author  : Kenny-PC
+@Software: PyCharm
+
+# code is far away from bugs with the god animal protecting
+    I love animals. They taste delicious.
+              ┏┓      ┏┓
+            ┏┛┻━━━┛┻┓
+            ┃      ☃      ┃
+            ┃  ┳┛  ┗┳  ┃
+            ┃      ┻      ┃
+            ┗━┓      ┏━┛
+                ┃      ┗━━━┓
+                ┃  神兽保佑    ┣┓
+                ┃　永无BUG！   ┏┛
+                ┗┓┓┏━┳┓┏┛
+                  ┃┫┫  ┃┫┫
+                  ┗┻┛  ┗┻┛
+"""
+
+import time
+
+from LoggerService import LoggerService
+# import account_list as al
+from util import date_util
+
+
+def start_order_job(log):
+    log.info('start_order_job')
+
+
+start_order_time = '2020-09-21 16:05:00'
+start_cost_time = '2020-09-21 16:35:00'
+
+if __name__ == '__main__':
+    log = LoggerService.logger_file('abc.log', 'abcd')
+
+    print(int(time.time()))
+    print(date_util.getCurrentSecondTime())
+
+    # print(date_util.getYesterdayStartTime(), platform_util.getZzyQueryTime(date_util.getYesterdayStartTime()))
+
+    request_time_stamp = time.time()
+    st_unix = int((request_time_stamp + 8 * 3600) // 86400 * 86400 - 8 * 3600 - 86400)  # 昨天开始时间
+    et_unix = int((request_time_stamp + 8 * 3600) // 86400 * 86400 - 8 * 3600)  # 昨天结束时间==(今天开始时间-1)
+    st_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(st_unix)) + '+08:00'
+    et_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(et_unix)) + '+08:00'
+    print(st_unix, st_dt)
+    print(et_unix, et_dt)
+
+    et_unix = et_unix - 1
+    print('========')
+    print((et_unix - st_unix))
+    timespace = 90 * 3600 * 24
+    print(timespace)
+    print((et_unix - st_unix) // timespace + 1)
+    for x in range((et_unix - st_unix) // timespace + 1):  # 分时段
+        print('x=', x)
+
+    print((et_unix - st_unix) / 86400 + 1)
+    for i in range(int((et_unix - st_unix) / 86400 + 1)):
+        print('i=', i)
+    for y in range(1):  # 分时段
+        print('y=', y)
+
+# print('时间转换')
+# print(date_util.getYesterdayStartTime(), date_util.getSecondsToDatetime(date_util.getYesterdayStartTime()))
+# print(date_util.getYesterdayEndTime(), date_util.getSecondsToDatetime(date_util.getYesterdayEndTime()))
+# print(date_util.getTodayStartTime(), date_util.getSecondsToDatetime(date_util.getTodayStartTime()))
+#
+# print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(date_util.getYesterdayStartTime())))
+# print(date_util.getCurrentSecondTime())
+# print(date_util.getSecondsToDatetime())
+# print(date_util.getSecondsToDatetime(date_util.getYesterdayStartTime()))
+# print(date_util.getSecondsToDatetime(date_util.getYesterdayStartTime(), "%Y-%m-%d %H:%M:%S"))
+
+# sql = "select * from quchen_text.order where date=1600790400"
+# connect = MySQLConnection('test')
+# try:
+# 	page = 1
+# 	size = 50
+# 	pagination = connect.listByPage(sql, page, size)
+# 	print(pagination)
+# except Exception as e:
+# 	log.exception(e)
+# finally:
+# 	# 关闭数据库连接
+# 	connect.close()
+
+# scheduler = BlockingScheduler()
+# scheduler.add_job(start_order_job, args=(log,), trigger='interval', max_instances=10, seconds=10,
+# 				  start_date=start_order_time)
+# scheduler.start()

dgp/tests/test_multiprocessing.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+__title__ = '测试多线程类'
+
+@Time    : 2020/9/26 22:53
+@Author  : Kenny-PC
+@Software: PyCharm
+
+# code is far away from bugs with the god animal protecting
+    I love animals. They taste delicious.
+              ┏┓      ┏┓
+            ┏┛┻━━━┛┻┓
+            ┃      ☃      ┃
+            ┃  ┳┛  ┗┳  ┃
+            ┃      ┻      ┃
+            ┗━┓      ┏━┛
+                ┃      ┗━━━┓
+                ┃  神兽保佑    ┣┓
+                ┃　永无BUG！   ┏┛
+                ┗┓┓┏━┳┓┏┛
+                  ┃┫┫  ┃┫┫
+                  ┗┻┛  ┗┻┛
+"""
+
+'''　　使用16线程爬取信息
+任务添加函数、任务执行函数；进程、线程切换函数；进、线程开启函数；
+'''
+import ssl
+from urllib import request, parse
+
+import requests
+
+ssl._create_default_https_context = ssl._create_unverified_context
+from datetime import datetime
+from multiprocessing import Pool as ProcessPoll  # 进程池
+from multiprocessing.dummy import Pool as ThreadPool  # 线程池
+
+import time
+import datetime
+import hashlib
+import json
+
+from util import date_util, platform_util
+import account_list as al
+
+zzy_order_list = ()
+
+
+def md5value(s):
+    md5 = hashlib.md5()
+    md5.update(s.encode("utf-8"))
+    return md5.hexdigest()
+
+
+# 任务执行
+def get_page(task_q):
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
+    }
+
+    # url action
+    qs = parse.parse_qs(task_q)
+    my_key = str(qs['a'][0])
+    secert = str(qs['b'][0])
+    stage = ''
+    if 'c' in qs.keys():
+        stage = qs['c'][0]
+
+    strlist = task_q.split('&a=')
+    task_q = strlist[0]
+    req = request.Request(task_q, headers=headers)
+
+    response = request.urlopen(req)
+
+    responseBodyStr = response.read().decode('utf8')
+    channel_list = json.loads(responseBodyStr)
+
+    # 掌中云的时间格式比较特殊,转换下
+    st = platform_util.getZzyQueryTime(date_util.getYesterdayStartTime())
+    et = platform_util.getZzyQueryTime(date_util.getYesterdayEndTime())
+
+    if 'data' in channel_list:
+        items = channel_list['data']['items']
+    # items = []
+    else:
+        print('channel_list', channel_list)
+        items = []
+
+    keyChildOrder = ()
+    for item in items:  # 获取channel_id 后逐个拉取历史orders
+        r = ()
+        channel_id = item['id']
+        channel = item['nickname']
+        status = str(1)
+        per_page = str(1000)
+        limit_time = et
+        get_time = st
+        lt = parse.urlencode({'created_at[lt]': limit_time})
+        gt = parse.urlencode({'created_at[gt]': get_time})
+        url_1 = 'https://openapi.818tu.com/partners/channel/orders/list?'
+        my_sign_1 = md5value(secert + 'channel_id=' + str(
+            channel_id) + '&created_at[gt]=' + get_time + '&created_at[lt]=' + limit_time + '&key=' + my_key + '&per_page=' + per_page + '&status=' + status)
+        parameter_1 = 'channel_id=' + str(
+            channel_id) + '&' + gt + '&' + lt + '&per_page=' + per_page + '&status=' + status + '&key=' + my_key + '&sign=' + my_sign_1
+        orders = requests.get(url_1 + parameter_1)
+        t = orders.json()['data']['count'] // int(per_page) + 1
+        for page in range(1, t + 1):
+            my_sign_2 = md5value(secert + 'channel_id=' + str(
+                channel_id) + '&created_at[gt]=' + get_time + '&created_at[lt]=' + limit_time + '&key=' + my_key + '&page=' + str(
+                page) + '&per_page=' + per_page + '&status=' + status)
+            parameter_2 = 'channel_id=' + str(channel_id) + '&' + gt + '&' + lt + '&page=' + str(
+                page) + '&per_page=' + per_page + '&status=' + status + '&key=' + my_key + '&sign=' + my_sign_2
+            orders_1 = requests.get(url_1 + parameter_2)
+            b = orders_1.json()['data']['items']
+
+            for a in b:
+                c = {}
+                c['user_id'] = str(a['member']['openid'])
+                c['channel'] = channel
+                c['reg_time'] = a['member']['created_at']
+                c['channel_id'] = channel_id
+                c['amount'] = round(a['price'] / 100, 2)
+                c['order_id'] = str(a['id'])
+                c['order_time'] = a['created_at']
+                c['platform'] = '掌中云'
+                c['stage'] = stage
+                # c['amount']=a['amount']
+                dtime = datetime.datetime.strptime(a['created_at'][0:10], "%Y-%m-%d")
+                c['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
+
+                if str(a['from_novel_id']) != 'None':
+                    c['from_novel'] = a['from_novel']['title']
+                else:
+                    c['from_novel'] = 'None'
+                """
+                del a['member']
+                del a['referral_link_id']
+                del a['id']
+                del a['created_at']
+                del a['paid_at']
+                del a['border_id']
+                del a['from_novel_id']
+                del a['status']
+                del a['price']
+                del a['agent_uid']
+                """
+                x = sorted(c.items(), key=lambda item: item[0])
+                x = dict(x)
+                x = tuple(x.values())
+                r = r + ((x),)
+
+        print('zzy_my_key_chanel', my_key, channel, len(r))
+        if len(r) > 0:
+            keyChildOrder = r + keyChildOrder
+
+    print('zzy_my_key:', my_key, ' 下的订单数量为：', len(keyChildOrder))
+    return keyChildOrder
+
+
+# 任务添加
+def url_list():
+    task_list = []
+
+    API_list = al.zzy_account_list
+    url = 'https://openapi.818tu.com/partners/channel/channels/list?'
+    for x in API_list:
+        my_key = x[0]
+        secert = x[1]
+        stage = x[2]
+        my_sign = md5value(secert + 'key=' + my_key)
+        # todo 这里一定要手动编码，坑爹的
+        my_key = parse.quote(x[0])
+        parameter = 'key=' + my_key + '&sign=' + my_sign
+        real_url = url + parameter
+        full_url = real_url + '&a=' + my_key + '&b=' + secert + '&c=' + stage
+        task_list.append(full_url)
+
+    return task_list
+
+
+# 设定进、线程
+def get_pool(way=True, count=4):
+    if way:
+        # 进程
+        pool = ProcessPoll(count)
+    else:
+        # 线程
+        pool = ThreadPool(count)
+    return pool
+
+
+# 启动
+def open_pool():
+    pool = get_pool(way=False, count=16)
+    task_q = url_list()
+
+    results = pool.map(get_page, task_q)  # 函数, 列表或元组
+    pool.close()
+    pool.join()
+    totalCount = 0
+    if len(results) > 0:
+        for item in results:
+            totalCount = totalCount + len(item)
+    print('掌中云中订单数据：', totalCount)
+
+
+if __name__ == '__main__':
+    start_second_time = date_util.getCurrentSecondTime()
+    open_pool()
+    print('执行时间：', date_util.getCurrentSecondTime() - start_second_time)

dgp/tests/test_pool.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+__title__ = '线程池测试'
+
+@Time    : 2020/9/27 13:57
+@Author  : zhengwangeng
+@Software: PyCharm
+
+# code is far away from bugs with the god animal protecting
+    I love animals. They taste delicious.
+              ┏┓      ┏┓
+            ┏┛┻━━━┛┻┓
+            ┃      ☃      ┃
+            ┃  ┳┛  ┗┳  ┃
+            ┃      ┻      ┃
+            ┗━┓      ┏━┛
+                ┃      ┗━━━┓
+                ┃  神兽保佑    ┣┓
+                ┃　永无BUG！   ┏┛
+                ┗┓┓┏━┳┓┏┛
+                  ┃┫┫  ┃┫┫
+                  ┗┻┛  ┗┻┛
+"""
+import os
+import random
+import time
+from concurrent.futures import ProcessPoolExecutor
+
+
+def task(n):
+    print('%s is runing' % os.getpid())
+    time.sleep(random.randint(1, 3))
+    return n
+
+
+if __name__ == '__main__':
+    print((100 - 1) // 100 + 1)
+    print((101 - 1) // 100 + 1)
+    for i in range((6 - 1) // 100 + 1):
+        print(i, ' == i')
+    for j in range((101 - 1) // 100 + 1):
+        print(j, ' == j')
+
+    executor = ProcessPoolExecutor(max_workers=3)
+
+    futures = []
+    for i in range(11):
+        future = executor.submit(task, i)
+        futures.append(future)
+    executor.shutdown(True)
+    print('+++>')
+    for future in futures:
+        print(future.result())

dgp/tests/test_threadpool.py

@@ -0,0 +1,638 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+__title__ = '每日凌晨空闲时检查本地数据库中的订单数据是否和平台昨天总订单一致'
+
+@Time    : 2020/9/26 19:44
+@Author  : Kenny-PC
+@Software: PyCharm
+
+# code is far away from bugs with the god animal protecting
+    I love animals. They taste delicious.
+              ┏┓      ┏┓
+            ┏┛┻━━━┛┻┓
+            ┃      ☃      ┃
+            ┃  ┳┛  ┗┳  ┃
+            ┃      ┻      ┃
+            ┗━┓      ┏━┛
+                ┃      ┗━━━┓
+                ┃  神兽保佑    ┣┓
+                ┃　永无BUG！   ┏┛
+                ┗┓┓┏━┳┓┏┛
+                  ┃┫┫  ┃┫┫
+                  ┗┻┛  ┗┻┛
+"""
+
+import datetime
+import hashlib
+import math
+import time
+from concurrent.futures import ProcessPoolExecutor
+from urllib import parse
+
+import requests
+
+import account_list as al
+from MySQLConnection import MySQLConnection
+from util import date_util, platform_util
+
+
+def md5value(s):
+    md5 = hashlib.md5()
+    md5.update(s.encode("utf-8"))
+    return md5.hexdigest()
+
+
+##《1》阅文
+def get_yuewen_order(st, et):
+    url = 'https://open.yuewen.com/cpapi/wxRecharge/querychargelog'
+    version = 1
+    secert_list = al.yuewen_account_list
+
+    for secert in secert_list:
+        start_time = st
+        email = secert[0]
+        appsecert = secert[1]
+
+        for i in range(int((et - st) / 86400)):
+            t = ()
+            end_time = min(start_time + 86400, et)
+            timestamp = int(time.time())
+            s = ''
+            page = 1
+            order_status = 2
+            data = {
+                'email': email,
+                'version': version,
+                'timestamp': timestamp,
+                'start_time': start_time,
+                'end_time': end_time,
+                'page': page,
+                'order_status': order_status
+                # 'last_min_id':last_min_id,
+                # 'last_max_id':last_max_id,
+                # 'total_count':total_count,
+                # 'last_page':last_page
+            }
+            sorted_data = sorted(data.items())
+            for k, v in sorted_data:
+                s = s + str(k) + str(v)
+
+            sign = md5value(appsecert + s).upper()
+
+            data1 = {
+                'email': email,
+                'version': version,
+                'timestamp': timestamp,
+                'start_time': start_time,
+                'end_time': end_time,
+                'page': page,
+                'order_status': order_status,
+                'sign': sign
+            }
+            list1 = requests.get(url=url, params=data1)
+
+            total_count = list1.json()['data']['total_count']
+            last_min_id = list1.json()['data']['min_id']
+            last_max_id = list1.json()['data']['max_id']
+            last_page = list1.json()['data']['page']
+
+            if total_count > 0:
+                for x in list1.json()['data']['list']:
+                    y = {}
+                    dtime = datetime.datetime.strptime(x['order_time'], "%Y-%m-%d %H:%M:%S")
+                    y['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
+                    y['platform'] = '阅文'
+                    y['channel'] = x['app_name']
+                    y['from_novel'] = x['book_name']
+                    y['user_id'] = x['openid']
+                    y['stage'] = ''
+                    y['channel_id'] = 0
+                    y['order_time'] = x['order_time']
+                    y['amount'] = x['amount']
+                    y['reg_time'] = x['reg_time']
+                    y['order_id'] = x['order_id']
+                    """
+                    del x['app_name']
+                    del x['order_status']
+                    del x['order_type']
+                    del x['openid']
+                    del x['user_name']
+                    del x['sex']
+                    del x['channel_name']
+                    del x['book_id']
+                    del x['book_name']
+                    del x['report_status']
+                    """
+                    y = sorted(y.items(), key=lambda item: item[0])
+                    y = dict(y)
+                    y = tuple(y.values())
+                    t = t + ((y),)
+
+            if total_count > 100:
+                for page in range(2, math.ceil(total_count / 100) + 1):
+                    data = {
+                        'email': email,
+                        'version': version,
+                        'timestamp': timestamp,
+                        'start_time': start_time,
+                        'end_time': end_time,
+                        'page': page,
+                        'last_min_id': last_min_id,
+                        'last_max_id': last_max_id,
+                        'total_count': total_count,
+                        'last_page': last_page,
+                        'order_status': order_status
+                    }
+                    sorted_data = sorted(data.items())
+                    s1 = ''
+                    for k, v in sorted_data:
+                        s1 = s1 + str(k) + str(v)
+                        sign = md5value(appsecert + s1).upper()
+                        data2 = {
+                            'email': email,
+                            'version': version,
+                            'timestamp': timestamp,
+                            'start_time': start_time,
+                            'end_time': end_time,
+                            'page': page,
+                            'last_min_id': last_min_id,
+                            'last_max_id': last_max_id,
+                            'total_count': total_count,
+                            'last_page': last_page,
+                            'order_status': order_status,
+                            'sign': sign
+                        }
+                    list2 = requests.get(url=url, params=data2)
+                    for x in list2.json()['data']['list']:
+                        y = {}
+                        dtime = datetime.datetime.strptime(x['order_time'], "%Y-%m-%d %H:%M:%S")
+                        y['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
+                        y['platform'] = '阅文'
+                        y['channel'] = x['app_name']
+                        y['from_novel'] = x['book_name']
+                        y['user_id'] = x['openid']
+                        y['stage'] = ''
+                        y['channel_id'] = 0
+                        y['order_time'] = x['order_time']
+                        y['amount'] = x['amount']
+                        y['reg_time'] = x['reg_time']
+                        y['order_id'] = x['order_id']
+                        """
+                        del x['app_name']
+                        del x['order_status']
+                        del x['order_type']
+                        del x['openid']
+                        del x['user_name']
+                        del x['sex']
+                        del x['channel_name']
+                        del x['book_id']
+                        del x['book_name']
+                        del x['report_status']
+                        """
+                        y = sorted(y.items(), key=lambda item: item[0])
+                        y = dict(y)
+                        y = tuple(y.values())
+                        t = t + ((y),)
+
+                    total_count = list2.json()['data']['total_count']
+                    last_min_id = list2.json()['data']['min_id']
+                    last_max_id = list2.json()['data']['max_id']
+                    last_page = list2.json()['data']['page']
+
+            print(email, start_time, len(t))
+            start_time = start_time + 86400
+            if len(t) > 0:
+                mysql_insert_order(t)
+
+
+##《2》掌读
+def get_zhangdu_order(st, et):
+    secert_list = al.zhangdu_account_list
+    url = 'https://api.zhangdu520.com/channel/getorder'
+
+    for item in secert_list:  # 分渠道
+        t = ()
+
+        uid = item[0]
+        appsecert = item[1]
+        channel = item[2]
+        timestamp = int(time.time())
+        sign = md5value(str(uid) + '&' + appsecert + '&' + str(timestamp))
+        page = 1
+        starttime = st
+        timespace = 90 * 3600 * 24
+        endtime = min(et, st + timespace)
+
+        for x in range((et - st) // timespace + 1):  # 分时段
+            Params = {
+                'uid': uid,
+                'timestamp': timestamp,
+                'sign': sign,
+                'starttime': starttime,
+                'endtime': endtime
+            }
+            list1 = requests.get(url=url, params=Params)
+            pageCount = list1.json()['data']['pageCount']
+            if pageCount > 0:
+                for a in range(1, pageCount + 1):  # 分页
+                    page = a
+                    Params = {
+                        'uid': uid,
+                        'timestamp': timestamp,
+                        'sign': sign,
+                        'starttime': starttime,
+                        'endtime': endtime,
+                        'page': page
+                    }
+                    list2 = requests.get(url=url, params=Params).json()
+                    if 'data' in list2.keys():
+                        for b in list2['data']['list']:
+                            c = {}
+                            c['amount'] = b['amount']
+                            c['channel_id'] = uid
+                            c['order_id'] = str(b['orderno'])
+                            c['order_time'] = b['ctime']
+                            c['user_id'] = b['openid']
+                            c['platform'] = '掌读'
+                            c['channel'] = channel
+                            c['reg_time'] = b['regtime']
+                            c['from_novel'] = ''
+                            c['stage'] = ''
+                            c['date'] = ((int(b['ctime']) + 8 * 3600) // 86400) * 86400 - 8 * 3600
+                            """
+                            del b['openid']
+                            del b['regtime']
+                            del b['ip']
+                            del b['ua']
+                            del b['id']
+                            del b['ctime']
+                            del b['userid']
+                            del b['orderno']
+                            del b['source']
+                            del b['sourceid']
+                            """
+                            if b['status'] == '1':
+                                # del b['status']
+                                del b
+                                x = sorted(c.items(), key=lambda item: item[0])
+                                x = dict(x)
+                                x = tuple(x.values())
+                                t = t + ((x),)
+                    else:
+                        print(list2)
+            starttime = starttime + timespace
+            endtime = min(et, starttime + timespace)
+        if len(t) > 0:
+            mysql_insert_order(t)
+        print('掌读', channel, len(t))
+
+
+##《3》花生
+def get_huasheng_order(st, et):
+    apikey_list = al.huasheng_account_list
+    url = 'https://vip.rlcps.cn/api/getMerchants'
+
+    for key in apikey_list:  # 获取每个vip账号下的channel_id
+        apiKEY = key[0]
+        apiSecurity = key[1]
+        stage = key[2]
+        timestamp = str(int(time.time()))
+        sign = md5value(apiKEY + timestamp + apiSecurity).upper()
+        data = {
+            'apiKey': apiKEY,
+            'apiSecurity': apiSecurity,
+            'timestamp': timestamp,
+            'sign': sign
+        }
+        list0 = requests.post(url, data).json()
+        t = ()
+
+        for merchant in list0['data']:
+            merchant_id = merchant['merchant_id']
+            merchant_name = merchant['merchant_name']
+            url1 = 'https://vip.rlcps.cn/api/orderList'
+            start_time = st
+
+            for i in range((et - st) // 86400):
+                page = 1
+                date = time.strftime("%Y-%m-%d", time.localtime(start_time))
+                sign = md5value(apiKEY + date + str(merchant_id) + timestamp + apiSecurity).upper()
+                data1 = {
+                    'apiKey': apiKEY,
+                    'apiSecurity': apiSecurity,
+                    'timestamp': timestamp,
+                    'date': date,
+                    'merchant_id': merchant_id,
+                    'sign': sign,
+                    'page': page
+                }
+                list1 = requests.post(url1, data1).json()
+
+                if 'data' in list1.keys() and len(list1['data']) > 0:
+
+                    for i in range(int(math.ceil(list1['count'] / 500))):
+                        data2 = {
+                            'apiKey': apiKEY,
+                            'apiSecurity': apiSecurity,
+                            'timestamp': timestamp,
+                            'date': date,
+                            'merchant_id': merchant_id,
+                            'sign': sign,
+                            'page': page
+                        }
+                        list2 = requests.post(url1, data2).json()
+
+                        for x in list2['data']:
+                            if x['order_status'] == 1:
+                                y = {}
+                                ##dtime = datetime.datetime.strptime(x['pay_at'],"%Y-%m-%d")
+                                ##y['date']= ((int(time.mktime(dtime.timetuple()))+8*3600)//86400)*86400-8*3600
+                                y['user_id'] = x['openid']
+                                y['order_id'] = x['trans_id']
+                                y['order_time'] = x['pay_at']
+                                y['reg_time'] = x['join_at']
+                                y['date'] = (start_time + 8 * 3600) // 86400 * 86400 - 8 * 3600
+                                y['channel'] = merchant_name
+                                y['channel_id'] = merchant_id
+                                y['platform'] = '花生'
+                                y['stage'] = stage
+                                y['from_novel'] = x['book_name']
+                                y['amount'] = x['amount']
+                                """
+                                del x['order_num']
+                                del x['book_name']
+                                del x['trans_id']
+                                del x['pay_at']
+                                del x['join_at']
+                                del x['subscribe_at']
+                                del x['openid']
+                                del x['charge_count']
+                                del x['book_id']
+                                del x['order_status']
+                                del x['user_name']
+                                del x['spread_name']
+                                del x['request_at']
+                                """
+                                y = sorted(y.items(), key=lambda item: item[0])
+                                y = dict(y)
+                                y = tuple(y.values())
+                                t = t + ((y),)
+                        page = page + 1
+                else:
+                    print(list1)
+                start_time = start_time + 86400
+
+        if len(t) > 0:
+            mysql_insert_order(t)
+    print(stage, merchant_name, len(t))
+
+
+##《4》掌中云
+def get_zzy_order(st, et):
+    # 掌中云的时间格式比较特殊,转换下
+    st = platform_util.getZzyQueryTime(st)
+    et = platform_util.getZzyQueryTime(et)
+
+    API_list = al.zzy_account_list
+    url = 'https://openapi.818tu.com/partners/channel/channels/list?'
+
+    for x in API_list:
+        my_key = x[0]
+        secert = x[1]
+        stage = x[2]
+        my_sign = md5value(secert + 'key=' + my_key)
+        parameter = 'key=' + my_key + '&sign=' + my_sign
+        channel_list = requests.get(url + parameter)  # 获取子渠道列表
+
+        if 'data' in channel_list.json().keys():
+            items = channel_list.json()['data']['items']
+        elif len(x) > 3:
+            my_key = x[3]
+            secert = x[4]
+            my_sign = md5value(secert + 'key=' + my_key)
+            parameter = 'key=' + my_key + '&sign=' + my_sign
+            channel_list = requests.get(url + parameter)
+            items = channel_list.json()['data']['items']
+        else:
+            print(channel_list.json())
+            items = []
+
+        for item in items:  # 获取channel_id 后逐个拉取历史orders
+            r = ()
+            channel_id = item['id']
+            channel = item['nickname']
+            status = str(1)
+            per_page = str(1000)
+            limit_time = et
+            get_time = st
+            lt = parse.urlencode({'created_at[lt]': limit_time})
+            gt = parse.urlencode({'created_at[gt]': get_time})
+            url_1 = 'https://openapi.818tu.com/partners/channel/orders/list?'
+            my_sign_1 = md5value(secert + 'channel_id=' + str(
+                channel_id) + '&created_at[gt]=' + get_time + '&created_at[lt]=' + limit_time + '&key=' + my_key + '&per_page=' + per_page + '&status=' + status)
+            parameter_1 = 'channel_id=' + str(
+                channel_id) + '&' + gt + '&' + lt + '&per_page=' + per_page + '&status=' + status + '&key=' + my_key + '&sign=' + my_sign_1
+            orders = requests.get(url_1 + parameter_1)
+            t = orders.json()['data']['count'] // int(per_page) + 1
+            for page in range(1, t + 1):
+                my_sign_2 = md5value(secert + 'channel_id=' + str(
+                    channel_id) + '&created_at[gt]=' + get_time + '&created_at[lt]=' + limit_time + '&key=' + my_key + '&page=' + str(
+                    page) + '&per_page=' + per_page + '&status=' + status)
+                parameter_2 = 'channel_id=' + str(channel_id) + '&' + gt + '&' + lt + '&page=' + str(
+                    page) + '&per_page=' + per_page + '&status=' + status + '&key=' + my_key + '&sign=' + my_sign_2
+                orders_1 = requests.get(url_1 + parameter_2)
+                b = orders_1.json()['data']['items']
+
+                for a in b:
+                    c = {}
+                    c['user_id'] = str(a['member']['openid'])
+                    c['channel'] = channel
+                    c['reg_time'] = a['member']['created_at']
+                    c['channel_id'] = channel_id
+                    c['amount'] = round(a['price'] / 100, 2)
+                    c['order_id'] = str(a['id'])
+                    c['order_time'] = a['created_at']
+                    c['platform'] = '掌中云'
+                    c['stage'] = stage
+                    # c['amount']=a['amount']
+                    dtime = datetime.datetime.strptime(a['created_at'][0:10], "%Y-%m-%d")
+                    c['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
+
+                    if str(a['from_novel_id']) != 'None':
+                        c['from_novel'] = a['from_novel']['title']
+                    else:
+                        c['from_novel'] = 'None'
+                    """
+                    del a['member']
+                    del a['referral_link_id']
+                    del a['id']
+                    del a['created_at']
+                    del a['paid_at']
+                    del a['border_id']
+                    del a['from_novel_id']
+                    del a['status']
+                    del a['price']
+                    del a['agent_uid']
+                    """
+                    x = sorted(c.items(), key=lambda item: item[0])
+                    x = dict(x)
+                    x = tuple(x.values())
+                    r = r + ((x),)
+
+            if len(r) > 0:
+                mysql_insert_order(r)
+
+            print('zzy', channel, len(r))
+
+
+##《5》 悠书阁
+def get_ysg_order(st, et):
+    start_exec_seconds = date_util.getCurrentSecondTime()
+    total_order = ()
+    account_list = al.ysg_account_list
+
+    executor = ProcessPoolExecutor(max_workers=4)
+
+    futures = []
+    for account in account_list:
+        future = executor.submit(get_ysg_order_task, st, et, account)
+        futures.append(future)
+    executor.shutdown(True)
+
+    for future in futures:
+        if len(future.result()) > 0:
+            total_order = future.result() + total_order
+
+    print('悠书阁订单数量：', len(total_order))
+    print('悠书阁订单执行时间（秒）：', date_util.getCurrentSecondTime() - start_exec_seconds)
+    return total_order
+
+
+def get_ysg_order_task(st, et, account):
+    total_order = ()
+    url = 'https://novel.youshuge.com/v2/open/orders'
+    # 超过100条就需要分页，别问我为什么知道，看代码看出来的
+    max_page_size = 100
+
+    host_name = account[0]
+    channel_id = account[1]
+    secert_key = account[2]
+    channel = account[3]
+    stage = account[4]
+
+    timestamp = int(time.time())
+    start_date = time.strftime("%Y-%m-%d", time.localtime(st))
+    end_date = time.strftime("%Y-%m-%d", time.localtime(et))
+    page = 1
+    str1 = 'channel_id=' + str(channel_id) + '&end_date=' + end_date + '&host_name=' + host_name + '&page=' + str(
+        page) + '&pay_status=1' + '&start_date=' + start_date + '&time=' + str(timestamp) + '&key=' + secert_key
+    sign = md5value(str1).upper()
+    data = {
+        'sign': sign,
+        'host_name': host_name,
+        'time': timestamp,
+        'channel_id': channel_id,
+        'page': page,
+        'pay_status': 1,
+        'start_date': start_date,
+        'end_date': end_date
+    }
+    result_json = requests.post(url, data).json()
+    first_page_order = build_ysg_order_data(channel, channel_id, result_json, stage)
+    total_order = total_order + first_page_order
+    if len(first_page_order) == 0:
+        return total_order
+
+    total_count = result_json['data'][0]['count']
+    if total_count > max_page_size:
+        for i in range((total_count - 1) // max_page_size + 1):
+            timestamp = int(time.time())
+            str1 = 'channel_id=' + str(
+                channel_id) + '&end_date=' + end_date + '&host_name=' + host_name + '&page=' + str(
+                page) + '&pay_status=1' + '&start_date=' + start_date + '&time=' + str(timestamp) + '&key=' + secert_key
+            sign = md5value(str1).upper()
+            data2 = {
+                'sign': sign,
+                'host_name': host_name,
+                'time': timestamp,
+                'channel_id': channel_id,
+                'page': page,
+                'pay_status': 1,
+                'start_date': start_date,
+                'end_date': end_date
+            }
+            r2 = requests.post(url, data2).json()
+
+            total_order = total_order + build_ysg_order_data(channel, channel_id, r2, stage)
+            page = page + 1
+
+    return total_order
+
+
+def build_ysg_order_data(channel, channel_id, result_json, stage):
+    order = ()
+    if 'data' in result_json.keys():
+        data = result_json['data']
+        if len(data) > 0:
+            for x in data:
+                y = {}
+                dtime = datetime.datetime.strptime(x['create_time'][0:10], "%Y-%m-%d")
+                y['date'] = ((int(
+                    time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
+                y['order_id'] = x['order_num']
+                y['amount'] = round(int(x['price']) / 100, 2)
+                y['order_time'] = x['create_time']
+                y['channel'] = channel
+                y['from_novel'] = x['book_name']
+                y['stage'] = stage
+                y['user_id'] = x['openid']
+                y['channel_id'] = channel_id
+                y['platform'] = '悠书阁'
+                y['reg_time'] = x['reg_time']
+
+                y = sorted(y.items(), key=lambda item: item[0])
+                y = dict(y)
+                y = tuple(y.values())
+                order = order + ((y),)
+    return order
+
+
+## 数据导入表采用replace替换主键orderid的方法
+def mysql_insert_order(data):
+    if len(data) == 0:
+        print('数据为空，不执行数据库操作！')
+        pass
+
+    sql = 'replace into quchen_text.order1 (amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
+    connect = MySQLConnection('test')
+    try:
+        num = connect.batch(sql, data)
+        print(num, '条订单数据入库成功')
+    except Exception as e:
+        print('订单数据入库失败：', e)
+    finally:
+        connect.close()
+
+
+def start_all_job():
+    st_unix = date_util.getYesterdayStartTime()
+    et_unix = date_util.getTodayStartTime()
+
+    print(date_util.getSecondsToDatetime(st_unix))
+    print(date_util.getSecondsToDatetime(et_unix))
+
+    start_exec_seconds = date_util.getCurrentSecondTime()
+    # get_yuewen_order(st_unix, et_unix)
+    ysg_order_list = get_ysg_order(st_unix, et_unix)
+    mysql_insert_order(ysg_order_list)
+
+    # get_zhangdu_order(st_unix, et_unix)
+
+    # get_zzy_order(st_unix, et_unix)
+    # get_huasheng_order(st_unix, et_unix)
+
+
+if __name__ == '__main__':
+    start_all_job()

dgp/tests/util/date_util.py

@@ -0,0 +1,302 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+__title__ = '操作时间的工具类'
+
+@Time    : 2020/9/26 19:44
+@Author  : Kenny-PC
+@Software: PyCharm
+
+# code is far away from bugs with the god animal protecting
+    I love animals. They taste delicious.
+              ┏┓      ┏┓
+            ┏┛┻━━━┛┻┓
+            ┃      ☃      ┃
+            ┃  ┳┛  ┗┳  ┃
+            ┃      ┻      ┃
+            ┗━┓      ┏━┛
+                ┃      ┗━━━┓
+                ┃  神兽保佑    ┣┓
+                ┃　永无BUG！   ┏┛
+                ┗┓┓┏━┳┓┏┛
+                  ┃┫┫  ┃┫┫
+                  ┗┻┛  ┗┻┛
+"""
+
+import datetime
+import time
+
+
+# ==========================
+# ========== time ==========
+# ==========================
+
+
+def getCurrentMilliSecondTime():
+    """
+    description:  获取当前时间-毫秒级
+    return:       1557730376981 -> str
+    """
+    timestamps = str(round(time.time() * 1000))
+    return timestamps
+
+
+def getCurrentSecondTime():
+    """
+    description:  获取当前时间-秒级
+    return:       1557730377 -> int
+    """
+    timestamps = int(time.time())
+    return timestamps
+
+
+def getCurrentTimeTuple(times=time.time()):
+    """
+    description:  接受秒级时间戳并返回时间元组（与mktime(tuple)相反）
+    times:        默认当前时间 可传second
+    return:       (tm_year=2019, tm_mon=5, tm_mday=13, tm_hour=10, tm_min=9, tm_sec=18, tm_wday=0, tm_yday=133, tm_isdst=0) -> tuple
+    tips:         time.localtime() 不传参则取当前时间
+    """
+    timestamps = time.localtime(times)
+    return timestamps
+
+
+def getTimeByTuple(tupleTime=time.localtime()):
+    """
+    description:  接受时间元组并返回秒级时间戳（与localtime(sec)相反）
+    tupleTime:    默认当前时间的元组 可通过time.localtime() or datetime.datetime.now().timetuple()获取
+    return:       1557733061 -> str
+    """
+    timestamps = str(round(time.mktime(tupleTime)))
+    return timestamps
+
+
+def getCurrentFormatTimeStr(times=time.time()):
+    """
+    description:  将指定时间元组格式化为字符串
+    times:        默认当前时间 可传second
+    return:       2019-05-13 15:00:47 -> str
+    tips:         %y 两位数的年份表示（00-99）    %Y 四位数的年份表示（000-9999）   %m 月份（01-12）    %d 月内中的一天（0-31）
+                  %H 24小时制小时数（0-23）      %I 12小时制小时数（01-12）        %M 分钟数（00=59）  %S 秒（00-59）   %w 星期（0-6）
+    """
+    timestamps = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(times))
+    return timestamps
+
+
+def getCurrentTimeTupleByFormatStr(time_str=str(datetime.datetime.now()).split(".")[0],
+                                   format_type="%Y-%m-%d %H:%M:%S"):
+    """
+    description:  接受格式化字符串返回时间元组
+    time_str:     格式化字符串   如:2019-05-13 15:00:47    默认当前时间
+    format_type:  格式化规则    如:%Y-%m-%d %H:%M:%S      默认%Y-%m-%d %H:%M:%S
+    return:       (tm_year=2019, tm_mon=5, tm_mday=13, tm_hour=10, tm_min=9, tm_sec=18, tm_wday=0, tm_yday=133, tm_isdst=0) -> tuple
+    """
+    return time.strptime(time_str, format_type)
+
+
+def getCurrentTimeStr():
+    """
+    description:  获取当前时间的可读形式字符串
+    return:       Mon May 13 11:27:42 2019 -> str
+    """
+    return time.ctime()
+
+
+def getCurrentTimeStrByTuple(tupleTime=time.localtime()):
+    """
+    description:  获取指定时间的可读形式字符串
+    tupleTime:    时间元组 可通过time.localtime() or datetime.datetime.now().timetuple()获取 默认当前时间的元组
+    return:       Mon May 13 11:27:42 2019 -> str
+    """
+    return time.asctime(tupleTime)
+
+
+def sleepTime():
+    """
+    description:  推迟调用线程的运行
+    """
+    for i in range(4):
+        print(i)
+        time.sleep(3)
+
+
+# ======================
+# ====== datetime ======
+# ======================
+
+
+def getNowDateTime():
+    """
+    description:  获取当前日期&时间
+    return:       2019-05-13 14:41:15 -> str
+    """
+    timestamps = str(datetime.datetime.now()).split(".")[0]
+    return timestamps
+
+
+def getNowTime():
+    """
+    description:  获取当前时间
+    return:       14:41:15 -> str
+    """
+    timestamps = str(datetime.datetime.now().time()).split(".")[0]
+    return timestamps
+
+
+def getTodayDate():
+    """
+    description:  获取当前日期
+    return:       2019-05-13 -> str
+    tipe:         datetime.datetime.now().date()有相同效果
+    """
+    timestamps = str(datetime.date.today())
+    return timestamps
+
+
+def getTimeDate(times=time.time()):
+    """
+    description:  获取指定时间戳的日期
+    time:         秒 默认当前时间
+    return:       2019-05-13 -> str
+    tips:         一天86400秒
+    """
+    timestamps = str(datetime.date.fromtimestamp(round(times)))
+    return timestamps
+
+
+def getAnyDateTime(day, hour=0, min=0, sec=0):
+    """
+    description:  获取距离现在时间的任意时间的日期&时间
+    day:          天数 1代表当前时间+1天    -1代表当前时间-1天
+    hour:         小时 2代表当前时间+2h     -2代表当前时间-2h     默认=0
+    min:          分钟 30代表当前时间+30min -30代表当前时间-30m   默认=0
+    sec:          秒   120代表当前时间+120s -120代表当前时间-120s 默认=0
+    return:       2019-05-15 15:37:41 -> str
+    """
+    return str(datetime.datetime.now() + datetime.timedelta(days=day, hours=hour, minutes=min, seconds=sec)).split(".")[
+        0]
+
+
+def getSecondsToDatetime(seconds=getCurrentSecondTime(), datetime_format="%Y-%m-%d %H:%M:%S"):
+    """
+    description:      秒/时间戳转日期字符串
+    seconds:          时间戳 默认为当前时间戳
+    datetime_format:  格式字符串     默认=%Y-%m-%d %H:%M:%S
+    return:       2019-05-15 15:37:41 -> str
+    """
+    return time.strftime(datetime_format, time.localtime(seconds))
+
+
+def getAnyDateSecondTime(day, hour=0, min=0, sec=0):
+    """
+    description:  获取距离现在时间的任意时间的秒数
+    day:          天数 1代表当前时间+1天    -1代表当前时间-1天
+    hour:         小时 2代表当前时间+2h     -2代表当前时间-2h     默认=0
+    min:          分钟 30代表当前时间+30min -30代表当前时间-30m   默认=0
+    sec:          秒   120代表当前时间+120s -120代表当前时间-120s 默认=0
+    return:       1557902182 -> str
+    """
+    anyDay = datetime.datetime.now() + datetime.timedelta(days=day, hours=hour, minutes=min, seconds=sec)
+    return str(round(time.mktime(anyDay.timetuple())))
+
+
+def getTodayStartTime():
+    """
+    description:  获取当天0点的时间戳
+    return:       1557676800 -> str
+    """
+    return int(time.mktime(datetime.date.today().timetuple()))
+
+
+def getTodayEndTime():
+    """
+    description:  获取今天23点59分59秒的时间戳
+    return:       1601049599 -> str
+    """
+    yesterday = datetime.date.today() + datetime.timedelta(days=1)
+    return int(time.mktime(time.strptime(str(yesterday), '%Y-%m-%d'))) - 1
+
+
+def getYesterdayStartTime():
+    """
+    description:  获取昨天0点的时间戳
+    return:       1557676800 -> int
+    """
+    yesterday = datetime.date.today() - datetime.timedelta(days=1)
+    return int(time.mktime(time.strptime(str(yesterday), '%Y-%m-%d')))
+
+
+def getYesterdayEndTime():
+    """
+    description:  获取昨天23点59分59秒的时间戳
+    return:       1601049599 -> int
+    """
+    return int(time.mktime(time.strptime(str(datetime.date.today()), '%Y-%m-%d'))) - 1
+
+
+def getTomorrowStartTime():
+    """
+    description:  获取明天0点的时间戳
+    return:       1557676800 -> int
+    """
+    tomorrow = datetime.date.today() + datetime.timedelta(days=1)
+    return int(time.mktime(time.strptime(str(tomorrow), '%Y-%m-%d')))
+
+
+def getTomorrowEndTime():
+    """
+    description:  获取明天23点59分59秒的时间戳
+    return:       1601049599 -> str
+    """
+    yesterday = datetime.date.today() + datetime.timedelta(days=2)
+    return int(time.mktime(time.strptime(str(yesterday), '%Y-%m-%d'))) - 1
+
+
+def getCurrentWeekTime():
+    """
+    description:  获取本周周一0点
+    return:       1557676800 -> str
+    tips:         可替换成: timestamps = time.mktime(time.strptime(time.strftime("%Y-%m-%d", time.localtime(times)), "%Y-%m-%d"))
+    """
+    week = int(time.strftime("%w", time.localtime()))
+    times = round(time.time()) - (week - 1) * 86400
+    timestamps = time.mktime(datetime.date.fromtimestamp(times).timetuple())
+    return str(round(timestamps))
+
+
+def test():
+    print(getCurrentMilliSecondTime())
+    print(getCurrentSecondTime())
+    print(getCurrentFormatTimeStr())
+    print(getCurrentTimeTupleByFormatStr())
+    print("=======")
+    print(getCurrentTimeStr())
+    print(getCurrentTimeStrByTuple(time.localtime()))
+    print(getTimeByTuple(time.localtime()))
+    print("=======")
+    print(getNowDateTime())
+    print(getNowTime())
+    print(getNowDateTime())
+    print(getTodayDate())
+    print(getTimeDate(time.time() - 86400))
+    print("=======")
+    print(getAnyDateTime(2))
+    print(getAnyDateSecondTime(2))
+    print("=======")
+    print(getTodayStartTime())
+    print(getCurrentWeekTime())
+    print('昨天')
+    print(getYesterdayStartTime(), time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(getYesterdayStartTime())))
+    print(getYesterdayEndTime(), time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(getYesterdayEndTime())))
+    print('今天')
+    print(getTodayStartTime(), time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(getTodayStartTime())))
+    print(getTodayEndTime(), time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(getTodayEndTime())))
+    print('明天')
+    print(getTomorrowStartTime(), time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(getTomorrowStartTime())))
+    print(getTomorrowEndTime(), time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(getTomorrowEndTime())))
+    return '测试完毕！'
+
+
+if __name__ == '__main__':
+    print(test())

dgp/__init__.py → dgp/tests/util/platform_util.py

@@ -1,10 +1,13 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+
 """
-__title__ = 'Package_name' __author__ = '
-Package_name' __author__ = '
-USER'
-__mtime__ = '2020/9/24'
+__title__ = '平台操作工具类'
+
+@Time    : 2020/9/26 21:51
+@Author  : Kenny-PC
+@Software: PyCharm
+
 # code is far away from bugs with the god animal protecting
     I love animals. They taste delicious.
               ┏┓      ┏┓
@@ -20,3 +23,13 @@ __mtime__ = '2020/9/24'
                   ┃┫┫  ┃┫┫
                   ┗┻┛  ┗┻┛
 """
+
+import time
+
+
+def getZzyQueryTime(st_unix):
+    """
+    description:  掌中云的时间格式比较，需要转换下
+    return:       2020-09-25T00:00:00+08:00 -> str
+    """
+    return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(st_unix)) + '+08:00'

requirements.txt

@@ -1,5 +1,14 @@
-requests==2.24.0
-apscheduler==3.6.3
+APScheduler==3.6.3
+certifi==2020.6.20
+chardet==3.0.4
+DBUtils==1.3
+idna==2.10
 numpy==1.19.2
 pandas==1.1.2
-pymysql==0.10.1
+PyMySQL==0.10.1
+python-dateutil==2.8.1
+pytz==2020.1
+requests==2.24.0
+six==1.15.0
+tzlocal==2.1
+urllib3==1.25.10

todo_list.md

@@ -0,0 +1,10 @@
+# 后续优化点
+
+#### 配置优化，迁移到配置文件中
+* 数据库连接需要单独配置
+* 账号，token配置也需要独立成配置文件，方便实时获取最新的数据，而不需要重启脚本
+* 多线程，异步获取订单数据，目前获取数据都是单线程，太慢了，掌中云2020-09-25的订单数据共12106条，执行时间为529秒，需要特别优化。
+
+
+
+