1

app/etl/data_stat_daily.py

@@ -34,7 +34,7 @@ def run_order_history():
         ods_order(i)
         order_sync_ck(i)
         dw_channel_daily_total(i)
-    dw_daily_channel()
+    # dw_daily_channel()
     print('ok')
 
 
@@ -48,10 +48,14 @@ def run_cost_history():
         dw_daily_channel_cost(i)
         dw_order_channel_cost_sync_ck(i)
         dw_channel_daily_total(i)
-    dw_daily_channel()
+    # dw_daily_channel()
 
 import sys
 if __name__ == '__main__':
+
+    run_cost_history()
+    exit()
+
     if len(sys.argv)==1:
         run(yestoday)
     else:

app/etl/data_stat_hourly.py

@@ -5,6 +5,8 @@ from data_stat_task import *
 from sync_to_ck_task import *
 from app.etl.dm.dm_pitcher_daily_overview import dm_pitcher_daily_overview
 from app.etl.dm.dm_channel_summary import dm_channel_summary
+from app.etl.dw.dw_channel_daily import dw_daily_channel
+from app.etl.dw.dw_pitcher_daily import dw_pitcher_daily
 log=logger()
 du = DateUtils()
 yestoday = du.get_n_days(-1)
@@ -20,20 +22,21 @@ def do_cost():
     dw_daily_channel_cost(today)
     dw_order_channel_cost_sync_ck(today)
     dw_channel_daily_total(today)
-    dw_daily_pitcher(today)
+
 
 
 def main():
     do_order()
     do_cost()
     dw_daily_channel()
+    dw_pitcher_daily()
     dm_pitcher_daily_overview()
     dm_channel_summary()
 
 
 if __name__ == '__main__':
     main()
-
+    # do_cost()
 
 
 

app/etl/data_stat_task.py

@@ -147,61 +147,7 @@ def order_account_text():
 
 
 
-def dw_daily_channel():
-    """快照表 每日一更新  t-1"""
-    sql="""insert into dw_daily_channel
-select
-       dt,channel,pitcher,stage,platform,book,
-       order_count,order_user,order_amount,first_order_count,first_order_user,first_order_amount,
-       view_count,click_count,follow_user,cost,reg_order_count,reg_order_user,reg_order_amount,reg_order_amount30,
-       web_view_count,platform_view_count,web_order_count,total_cost,total_amount,reg_order_user_again,reg_order_user7,reg_order_user30,reg_order_amount7,type,
-       total_first_amount 
-from
-(select dt,channel, pitcher,stage,platform,book,cost,view_count,click_count,follow_user,web_view_count,platform_view_count,web_order_count,type from dw_daily_channel_cost) a
-    left outer join
-    (select toDate(formatDateTime(reg_time,'%Y-%m-%d')) as dt2,channel as channel2,
-       sum(amount) as reg_order_amount,
-       count(distinct user_id) as reg_order_user,
-       count(1) as reg_order_count,
-       sum(if(subtractDays(date, 30)>reg_time,0,amount)) as reg_order_amount30,
-       count(distinct if(subtractDays(date, 7)>reg_time,'',user_id))-1 reg_order_user7,
-       sum(if(subtractDays(date, 7)>reg_time,0,amount)) as reg_order_amount7,
-       count(distinct if(subtractDays(date, 30)>reg_time,'',user_id))-1 reg_order_user30
-       from order where reg_time>'2019-03-18 00:00:00' group by toDate(formatDateTime(reg_time,'%Y-%m-%d')),channel) b
-        on dt=dt2 and channel=channel2 left outer join
-     (select date as dt3,channel as channel3,
-     count(1) as first_order_count,
-        count(distinct user_id) as first_order_user,
-             sum(amount) as first_order_amount
-       from order where toDate(reg_time)=date  group by date,channel) c
-        on dt=dt3 and channel=channel3
-    left outer join
-    (select date as dt4,channel as channel4,
-     count(1) as order_count,
-        count(distinct user_id) as order_user,
-             sum(amount) as order_amount
-       from order  group by date,channel) d
-        on dt=dt4 and channel=channel4
-left outer join
-     (select dt dt5,channel channel5,total_cost,total_amount,total_first_amount from dw_channel_daily_total ) e
-         on dt=dt5 and channel=channel5
-
-left outer join (
-select sum(if(user_order_count>1,1,0)) reg_order_user_again,channel channel6,toDate(reg_date) dt6  from (
-select formatDateTime(reg_time,'%Y-%m-%d') reg_date,channel,count(1) user_order_count
-    from order group by formatDateTime(reg_time,'%Y-%m-%d') ,user_id,channel
-    ) x group by reg_date,channel
-    ) f on dt=dt6 and channel=channel6
-        """
-    ck.execute("truncate table dw_daily_channel")
-    ck.execute(sql)
-    print("ok")
-
-
-
-
-
-def dw_channel_daily_total(ymd):
+def   dw_channel_daily_total(ymd):
     sql=f"""insert into dw_channel_daily_total
             select '{ymd}' dt,channel,total_cost,total_amount,total_first_amount from
             (select channel,sum(cost) total_cost  from dw_daily_channel_cost where  dt<='{ymd}'  group by channel)a
@@ -216,34 +162,13 @@ def dw_channel_daily_total(ymd):
     print(sql)
     ck.execute(sql)
 
-def dw_daily_pitcher(ymd):
-    sql=f"""
-insert into dw_daily_pitcher
-select '{ymd}' dt, a.pitcher,
-       start_cost_date,end_cost_date,total_amount,total_cost
-from
-     (select sum(cost) total_cost,pitcher from dw_daily_channel_cost where dt<='{ymd}' group by pitcher) a
-left outer join
-    (select  min(dt) start_cost_date,max(dt) end_cost_date,pitcher
-        from dw_daily_channel_cost  where cost>0  group by pitcher) b on a.pitcher=b.pitcher
-left outer join
-    (
-    select b.pitcher pitcher,sum(if(toDate(formatDateTime(reg_time,'%Y-%m-%d'))>=start_cost_date,amount,0)) total_amount
-    from order a
-    left outer join dw_daily_channel_cost b on a.channel = b.channel and date=dt
-    left outer join (select  min(dt) start_cost_date,pitcher,channel from dw_daily_channel_cost  where cost>0   group by pitcher,channel)
-        c on b.pitcher=c.pitcher and b.channel=c.channel where date<='{ymd}' group by b.pitcher
-) d on a.pitcher=d.pitcher
-"""
-    print(sql)
-    ck.execute(f"alter table dw_daily_pitcher drop partition '{ymd}'")
-    ck.execute(sql)
+
 
 
 
 if __name__ == '__main__':
     # dw_daily_pitcher('2021-01-14')
-    dm_pitcher_daily_page_total()
+
     exit(0)
     # dw_channel_daily_total('2020-07-20')
     # channel_by_account_daily('2020-12-17')