1

app/api_data/get_order.py

@@ -83,10 +83,28 @@ def huasheng(start=None,end=None,new=None):
     save_hs_data(li)
 
 
-if __name__ == '__main__':
 
-    # yangguang('2021-01-16','2021-01-18')
-    huasheng('2021-01-18','2021-01-18')
+def hourly():
+    yangguang()
+    huasheng()
+
+
+def daily():
+    st = du.get_n_days(-10)
+    et = du.get_n_days(-1)
+    yangguang(et, et)
+    huasheng(et, et)
+
+    # 昨日新增账户 自动往前跑10天数据
+    yangguang(st, et, new=True)
+    huasheng(st, et, new=True)
+
+
+
+
+if __name__ == '__main__':
+    yangguang()
+    huasheng()
 
 
     """要是只跑一个账号 把 get_yg_acccount() 里面的sql where 条件加上 id=xxx"""

app/api_data/order_daily.py

@@ -1,10 +0,0 @@
-from app.api_data.get_order import *
-import time
-du=DateUtils()
-
-
-if __name__ == '__main__':
-    dt = du.get_n_days(-1)
-    huasheng(dt, dt)
-
-

app/api_data/order_hourly.py

@@ -1,22 +0,0 @@
-from app.api_data.get_order import *
-from model.DateUtils import DateUtils
-import time
-from model.DingTalkUtils import DingTalkUtils
-du=DateUtils()
-
-"""每小时5分跑"""
-
-if __name__ == '__main__':
-    try:
-        yangguang()
-        huasheng()
-    except:
-        DingTalkUtils.send('订单小时任务失败','18860455786')
-
-
-
-
-
-
-
-

app/api_data/order_run_back.py

@@ -1,12 +0,0 @@
-from app.api_data.get_order import *
-from model.DateUtils import DateUtils
-import time
-from model.DingTalkUtils import DingTalkUtils
-du=DateUtils()
-
-
-if __name__ == '__main__':
-    st = du.get_n_days(-10)
-    et = du.get_n_days(-1)
-    yangguang(st, et, new=True)
-    huasheng(st, et, new=True)

app/api_data/tg.py

@@ -0,0 +1,19 @@
+from model.DataBaseUtils import MysqlUtils
+db=MysqlUtils()
+with open('./zhangdu_account_config.csv','r',encoding='utf-8') as f:
+    # print(f.readlines())
+    for i in f.readlines():
+        a=  i.replace('\n','')
+        db.quchen_text.execute(f"insert into order_account_text(platform,text) values('掌读','{a}')")
+
+
+# a={'x':11,'y11':2}
+# for i in a:
+#     print(i)
+
+
+# a=['a','b']
+# b=[1,2]
+# print(dict(zip(a,b)))
+# get
+

app/dailyRun.py

@@ -0,0 +1,18 @@
+from app.api_data import get_order
+from app.etl import data_stat_run
+from model.DateUtils import DateUtils
+du = DateUtils()
+
+
+"""凌晨5点跑 为了防止数据提供方数据缺失情况"""
+
+if __name__ == '__main__':
+
+    # 订单数据
+    get_order.daily()
+
+    # 数据处理
+    data_stat_run.daily()
+
+
+

app/etl/data_stat_daily.py

@@ -1,77 +0,0 @@
-from model.DateUtils import DateUtils
-from data_stat_task import *
-from sync_to_ck_task import *
-du = DateUtils()
-from model.DingTalkUtils import DingTalkUtils
-from app.etl.dm.dm_pitcher_daily_overview import dm_pitcher_daily_overview
-yestoday = du.get_n_days(-1)
-today = du.get_n_days(0)
-"""凌晨5点跑昨天的数据"""
-
-
-def run(ymd):
-
-    try:
-        # 公众号元数据
-        channel_by_account_daily(ymd)
-        channel_info_daily(ymd)
-
-        # 清洗订单表-同步到ck
-        ods_order(ymd)
-        order_sync_ck(ymd)
-
-        # 公众号维度数据-同步到ck
-        dw_daily_channel_cost(ymd)
-        dw_order_channel_cost_sync_ck(ymd)
-        dw_channel_daily_total(ymd)
-    except Exception as e:
-        DingTalkUtils.send("数据ETL出错：",'18860455786')
-
-def run_order_history():
-    """当order表跑了历史的数据，要从这历史数据的最早时间跑此任务流"""
-    for i in du.getDateLists('2020-01-01',du.get_n_days(0)):
-        print(i)
-        ods_order(i)
-        order_sync_ck(i)
-        dw_channel_daily_total(i)
-    # dw_daily_channel()
-    print('ok')
-
-
-def run_cost_history():
-    "cost 重跑后，跑此任务流"
-    for i in du.getDateLists('2019-03-18',du.get_n_days(0)):
-        print(i)
-        channel_by_account_daily(i)
-        channel_info_daily(i)
-
-        dw_daily_channel_cost(i)
-        dw_order_channel_cost_sync_ck(i)
-        dw_channel_daily_total(i)
-    # dw_daily_channel()
-
-import sys
-if __name__ == '__main__':
-
-    run_cost_history()
-    exit()
-
-    if len(sys.argv)==1:
-        run(yestoday)
-    else:
-
-        for i in du.getDateLists(sys.argv[1],sys.argv[2]):
-            run(i)
-
-
-
-
-
-
-
-
-
-
-
-
-

app/etl/data_stat_hourly.py → app/etl/data_stat_run.py

@@ -9,34 +9,41 @@ from app.etl.dw.dw_channel_daily import dw_daily_channel
 from app.etl.dw.dw_pitcher_daily import dw_pitcher_daily
 log=logger()
 du = DateUtils()
-yestoday = du.get_n_days(-1)
-today = du.getTodayOrYestoday()
 
+import sys
 
-def do_order():
-    ods_order(today)
-    order_sync_ck(today)
-def do_cost():
-    channel_by_account_daily(today)
-    channel_info_daily(today)
-    dw_daily_channel_cost(today)
-    dw_order_channel_cost_sync_ck(today)
-    dw_channel_daily_total(today)
+def do_order(st,et):
+    for i in du.getDateLists(st,et):
+        ods_order(i)
+        order_sync_ck(i)
+def do_cost(st,et):
+    for i in du.getDateLists(st,et):
+        channel_by_account_daily(i)
+        channel_info_daily(i)
+        dw_daily_channel_cost(i)
+        dw_order_channel_cost_sync_ck(i)
+        dw_channel_daily_total(i)
 
 
-
-def main():
-    do_order()
-    do_cost()
+def main(st,et):
+    do_order(st,et)
+    do_cost(st,et)
     dw_daily_channel()
     dw_pitcher_daily()
     dm_pitcher_daily_overview()
     dm_channel_summary()
 
 
-if __name__ == '__main__':
-    main()
-    # do_cost()
+
+def hourly():
+    thedate=du.getTodayOrYestoday()
+    main(thedate, thedate)
+
+
+def daily():
+    dt=du.get_n_days(-1)
+    main(dt, dt)
+
 
 
 

app/hourlyRun.py

@@ -0,0 +1,18 @@
+
+from app.api_data import get_order
+from app.etl import data_stat_run
+from model.DateUtils import DateUtils
+du=DateUtils()
+
+
+
+if __name__ == '__main__':
+
+    # 订单数据
+    get_order.hourly()
+
+    # 数据处理
+    data_stat_run.hourly()
+
+
+

app/test.py

@@ -0,0 +1,30 @@
+import requests
+import json
+url = "https://download.yifengaf.cn/excel/2020-12-28/10008097/订单API数据495633-20201228162500.txt"
+
+# headers = {"Content-Type": "application/json"}
+
+r = requests.get(url,).text
+# print(r)
+# b=[]
+# a=r.split('}')
+# for i in a[:-1]:
+#     if i[-1]!='}':
+#         b.append(json.loads(i+"}",strict=False))
+#     else:
+#         b.append(json.loads(i,strict=False))
+
+# print(b)
+# 17382,18898,19369,18523,19115,29603,26878,29302,17544,26930,26932,26903,26883,29600,22761,22260,23009,23421,26873,19821,20844,20324,20326,21126,17866,20115,20844,20326,21126
+
+
+def q():
+    a="""{"merchant_id":"20201228235833_187292356_v5zy","transaction_id":,"type":"书币充值","money":"18.0","state":"未完成","create_time":"2020-12-28 23:58:34","finish_time":,"book_name":"神都猛虎","referral_url":,"user_id":"187292356","channel_id":"17228","subscribe_time":"","user_createtime":"","openid":"","ip":"","user_agent":""}"""
+    print(
+    json.loads(a)
+    )
+
+
+if __name__ == '__main__':
+    q()
+