1

app/api_data/platform_order/audio_qiyue.py

@@ -73,7 +73,9 @@ class AudioQiyue:
                         x['money'],
                         x['book_name'],
                         x['transaction_no'] if x['transaction_no'] != '0' else x['trade_no'],  # 订单id
-                        x['state'])
+                        x['state']
+                        # , x['user_id']
+                    )
                     )
                 next_page_url = result_data['next_page_url']
                 if next_page_url is None:
@@ -84,9 +86,8 @@ class AudioQiyue:
         print(f'{stage} [{start}~{end}] 有订单{order_list.__len__()}')
         if order_list.__len__() > 0:
             # print(order_list)
+            save_order(order_list)
 
-            save_order(db_operation=self.db_quchen_text, order_list=order_list)
-            # print(order_list)
 
 
 

app/api_data/get_order.py → app/api_data/platform_order/get_order.py

@@ -1,4 +1,4 @@
-from app.api_data.order_util import *
+from app.api_data.platform_order.order_util import *
 from model.DataBaseUtils import MysqlUtils
 from concurrent.futures import ThreadPoolExecutor
 from model.DingTalkUtils import DingTalkDecorators
@@ -6,7 +6,6 @@ from app.api_data.platform_order.audio_qiyue import AudioQiyue
 from app.api_data.platform_order.youshuge import get_youshuge_order_task
 from app.api_data.platform_order.yuewen import get_yuewen_order_task
 from app.api_data.platform_order.yangguang import yangguang
-from multiprocessing import Process
 
 db = MysqlUtils()
 

app/api_data/order_util.py → app/api_data/platform_order/order_util.py

@@ -14,6 +14,7 @@ db=MysqlUtils()
 du=DateUtils()
 
 
+
 def get_yg_vip_channel(stage, vip_id, client_id, token):
     url = "https://data.yifengaf.cn:443/channeldata/data/account/list"
     nonce = ComUtils.get_random_str()
@@ -102,7 +103,7 @@ def parse_yg_data(vip_id,stage):
     # print(insert_data)
     # exit(0)
     print("订单数："+ str(insert_data.__len__()))
-    db.quchen_text.executeMany("replace into ods_order values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",tuple(insert_data))
+    save_order(insert_data)
 
 
 def get_hs_order_task(start, end, account):
@@ -133,7 +134,7 @@ def get_hs_order_task(start, end, account):
     if len(li) > 0:
         print(f"花生账号：{account[2]} 有order{len(li)}个")
         # print(li)
-        db.quchen_text.executeMany("replace into ods_order values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",li)
+        save_order(li)
 
 
 
@@ -198,6 +199,8 @@ def get_huasheng_order(start,end, account, merchant):
                      i['book_name'],
                      i['trans_id'] if i['trans_id'] != '' else i['order_num'],
                      2 if i['order_status'] == 1 else 1
+                     # i['user_id']
+
                      )
                 )
 
@@ -268,7 +271,9 @@ def get_qiyue_order_task(start, end, account):
                      x['money'],
                      x['book_name'],
                      x['transaction_no'] if x['transaction_no'] != 0 else x['trade_no'],   # 订单id
-                     x['state'])
+                     x['state']
+                     # x['user_id']
+                )
                      )
 
             next_page_url = result_data['next_page_url']
@@ -422,6 +427,7 @@ def get_wd_order_task(start,end,account):
                      x['bookTitle'] if x['bookTitle'] else '',
                      x['ewTradeId'] if x.get('ewTradeId') else x['rechargeUuid'],
                      2 if x['payStatus'] == 1 else 1
+                     # ,x['userId']
                     )
                 )
             if len(order_item_list) < 1000:
@@ -489,6 +495,7 @@ def get_zd_order_task(start,end,account):
                     i['book_entry'],
                     i['orderno'],
                     2 if i['status'] == '1' else 1
+                    # ,i['userid']
 
                 ))
 
@@ -590,6 +597,7 @@ def get_zzy_channel_order(start, end, account, channel):
                 i['from_novel']['title'] if str(i['from_novel_id']) != 'None' else '',
                 str(i['id']),
                 2 if i['status'] == 1 else 1
+                # ,i['id']
             ))
 
 

app/api_data/platform_order/yangguang.py

@@ -1,4 +1,4 @@
-from app.api_data.order_util import *
+from app.api_data.platform_order.order_util import *
 from model.DataBaseUtils import MysqlUtils
 from model.DingTalkUtils import DingTalkDecorators,DingTalkUtils
 

app/api_data/platform_order/youshuge.py

@@ -2,10 +2,10 @@ import time
 from model.DateUtils import DateUtils
 from model.ComUtils import md5
 import requests
-from model.DataBaseUtils import MysqlUtils
-db = MysqlUtils()
+from app.api_data.platform_order.order_util import save_order
 ut = DateUtils()
 
+
 def get_youshuge_order_task(st, et, account):
 
     et = ut.add_days(et,1)
@@ -59,8 +59,8 @@ def get_youshuge_order_task(st, et, account):
 
     if len(li) > 0:
         print(f"{channel} 有订单{len(li)}")
-        db = MysqlUtils()
-        db.quchen_text.executeMany("replace into ods_order values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",li)
+        save_order(li)
+
 
 
 if __name__ == '__main__':

app/api_data/platform_order/yuewen.py

@@ -2,7 +2,7 @@ import time
 from model.DateUtils import DateUtils
 from model.ComUtils import md5
 import requests
-from model.DataBaseUtils import MysqlUtils
+from app.api_data.platform_order.order_util import save_order
 ut = DateUtils()
 
 def get_yuewen_order_task(st, et, account):
@@ -85,6 +85,7 @@ def get_yuewen_order_task(st, et, account):
                        i['book_name'],
                        i['order_id'],
                        i['order_status']
+
             ))
 
         if len(order_item_list) < 100:
@@ -94,8 +95,7 @@ def get_yuewen_order_task(st, et, account):
 
     if len(li) > 0:
         print(f"{email} 有订单{len(li)}")
-        db = MysqlUtils()
-        db.quchen_text.executeMany("replace into ods_order values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",li)
+        save_order(li)
 
 
 

app/api_data/tg.py

@@ -1,53 +0,0 @@
-# from model.DataBaseUtils import MysqlUtils
-# db=MysqlUtils()
-# with open('./zhangdu_account_config.csv','r',encoding='utf-8') as f:
-#     # print(f.readlines())
-#     for i in f.readlines():
-#         a=  i.replace('\n','')
-#         db.quchen_text.execute(f"insert into order_account_text(platform,text) values('掌读','{a}')")
-
-
-# a={'x':11,'y11':2}
-# for i in a:
-#     print(i)
-
-
-# a=['a','b']
-# b=[1,2]
-# print(dict(zip(a,b)))
-# get
-
-
-
-def zsq(func):
-    def x(*args,**kwargs):
-
-        try:
-            func(*args,**kwargs)
-        except Exception as e:
-            print(e)
-
-
-    return x
-
-
-def zsq2(info):
-    def wrapper(func):
-        def deco(*args, **kwargs):
-            try:
-                func(*args, **kwargs)
-            except Exception as e:
-                print(info, e)
-
-        return deco
-
-    return wrapper
-
-
-@zsq2("趣程")
-def a(p,q):
-    print(p/q)
-
-
-if __name__ == '__main__':
-    a(2,0)

app/api_data/tx_ad_cost/TxCostUtils.py

@@ -1,7 +1,5 @@
-
-from app.api_data.cost_util import *
 from model.DateUtils import DateUtils
-from model.DataBaseUtils import MysqlUtils
+
 du = DateUtils()
 
 # def get_accounts(filter=None):

app/api_data/get_cost.py → app/api_data/tx_ad_cost/get_cost.py

@@ -1,5 +1,5 @@
 
-from app.api_data.cost_util import *
+from app.api_data.tx_ad_cost.cost_util import *
 from model.DateUtils import DateUtils
 from model.DataBaseUtils import MysqlUtils
 from concurrent.futures import ThreadPoolExecutor

app/ad_hourly.py → app/crontab_task/ad_hourly.py

@@ -1,5 +1,5 @@
 from model.DateUtils import DateUtils
-from app.api_data import get_cost
+from app.api_data.tx_ad_cost import get_cost
 from app.etl.dw import dw_image_cost_day
 du = DateUtils()
 import time

app/daily08.py → app/crontab_task/daily08.py

@@ -1,6 +1,4 @@
-
-from model.DateUtils import DateUtils
-from app.task import *
+from app.crontab_task.task import *
 du = DateUtils()
 
 

app/dailyRun.py → app/crontab_task/dailyRun.py

@@ -1,9 +1,8 @@
-from app.api_data import get_order
 from app.etl import data_stat_run
 from model.DateUtils import DateUtils
-from app.api_data import get_cost
+from app.api_data.tx_ad_cost import get_cost
 from app.etl.dw import dw_image_cost_day
-from app import task
+from app.crontab_task import task
 from app.etl.MaterialLibrary import MaterialDataClean
 du = DateUtils()
 
@@ -23,7 +22,7 @@ if __name__ == '__main__':
     dw_image_cost_day.day()
 
     # 素材库
-    MaterialDataClean()
+    MaterialDataClean.run()
 
 
 

app/hourlyRun.py → app/crontab_task/hourlyRun.py

@@ -1,6 +1,7 @@
 from app.etl import data_stat_run
 from model.DateUtils import DateUtils
-from app import task
+from app.crontab_task import task
+
 du = DateUtils()
 import time
 from model.DingTalkUtils import DingTalkUtils

app/task.py → app/crontab_task/task.py

@@ -1,7 +1,7 @@
 
 import threading
-from app.api_data.get_order import *
-from app.api_data.get_cost_older import old_cost_hourly,old_cost_daily
+from app.api_data.platform_order.get_order import *
+from app.api_data.tx_ad_cost.get_cost_older import old_cost_hourly,old_cost_daily
 from app.etl.sync_to_ck_task import order_sync_ck
 
 def hourly():

app/etl/MaterialLibrary/MaterialDataClean.py

@@ -104,8 +104,12 @@ from dw_image_cost_day where signature is not null and signature!=''   GROUP BY
     db.zx_ads.dfsave2mysql(df, table, key, tag)
 
 
-if __name__ == '__main__':
+def run():
     title()
     description()
     image()
     adcreative()
+
+
+if __name__ == '__main__':
+    run()

app/test.py

@@ -1,30 +0,0 @@
-import requests
-import json
-url = "https://download.yifengaf.cn/excel/2020-12-28/10008097/订单API数据495633-20201228162500.txt"
-
-# headers = {"Content-Type": "application/json"}
-
-r = requests.get(url,).text
-# print(r)
-# b=[]
-# a=r.split('}')
-# for i in a[:-1]:
-#     if i[-1]!='}':
-#         b.append(json.loads(i+"}",strict=False))
-#     else:
-#         b.append(json.loads(i,strict=False))
-
-# print(b)
-# 17382,18898,19369,18523,19115,29603,26878,29302,17544,26930,26932,26903,26883,29600,22761,22260,23009,23421,26873,19821,20844,20324,20326,21126,17866,20115,20844,20326,21126
-
-
-def q():
-    a="""{"merchant_id":"20201228235833_187292356_v5zy","transaction_id":,"type":"书币充值","money":"18.0","state":"未完成","create_time":"2020-12-28 23:58:34","finish_time":,"book_name":"神都猛虎","referral_url":,"user_id":"187292356","channel_id":"17228","subscribe_time":"","user_createtime":"","openid":"","ip":"","user_agent":""}"""
-    print(
-    json.loads(a)
-    )
-
-
-if __name__ == '__main__':
-    q()
-

data_processing/sql_tools.py

@@ -1,5 +0,0 @@
-
-
-def save_order(db_operation,order_list):
-    db_operation.executeMany('replace into ods_order values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',order_list)
-    print("入库成功")

example/ding_test.py

@@ -1,2 +0,0 @@
-from model.DingTalkUtils import DingTalkUtils
-DingTalkUtils.send("这是测试文本", "18860455786")

example/insert_example.py

@@ -1,31 +0,0 @@
-"""
-Example
-pandas操作MySQL
-以pandas.dataframe 插入mysql
-"""
-from model.DataBaseUtils import MysqlUtils
-import pandas as pd
-db = MysqlUtils()
-
-
-li = [["1", "小米", "122", "南京"],
-      ["3", "小红", "11", "杭州"],
-      ["2", "小绿", "10", "上海"]]
-df = pd.DataFrame(li, columns=["id", "name", "age", "city"])
-
-
-key = ["id"]                                # 抽象主键,不管表里这个是不是主键,都会根据该值来决定是更新还是插入，但是若无该索引，会导致插入速度很慢
-tag = ["name", "age", "city"]               # 其他值
-
-db.beauty.insertorupdatemany_v2(
-    table='test',
-    keys=key,
-    tags=tag,
-    key_values=df[key],
-    tag_values=df[tag],
-    flag=False)
-
-# 关闭连接
-db.close()
-
-

example/select_example.py

@@ -1,18 +0,0 @@
-"""
-Example
-pandas操作MySQL
-
- 查数据，结果用dataframe显示
-"""
-from model.DataBaseUtils import MysqlUtils
-db = MysqlUtils()
-
-
-
-sql = "select * from notebook limit 10"
-df1 = db.beauty.getData_pd(sql)
-print(df1)
-
-
-# 关闭连接
-db.close()

example/selenium_model.py

@@ -1,15 +0,0 @@
-from selenium import webdriver
-import time
-option=webdriver.ChromeOptions()
-option.add_argument("""--user-data-dir=C:\\Users\\青山\\AppData\\Local\\Google\\Chrome\\User Data""")
-
-browser=webdriver.Chrome(executable_path='D:\workspace\chromedriver',chrome_options=option)
-
-
-browser.get('https://mp.weixin.qq.com/promotion/frame?t=ad_system/common_simple_frame&t1=campaign/edit&token=1340577178&from_pos_type=0#/')
-time.sleep(2)
-browser.find_element_by_xpath('//*[@id="kw"]').send_keys("selenium")
-time.sleep(2)
-browser.find_element_by_xpath('//*[@id="su"]').click()
-
-