1

app/etl/data_stat_daily.py

@@ -22,6 +22,7 @@ def run(ymd):
         # 公众号维度数据-同步到ck
         dw_daily_channel_cost(ymd)
         dw_order_channel_cost_sync_ck(ymd)
+        dw_channel_daily_total(ymd)
     except Exception as e:
         DingTalkUtils.send("数据ETL出错：",'18860455786')
 
@@ -31,6 +32,7 @@ def run_order_history():
         print(i)
         ods_order(i)
         order_sync_ck(i)
+        dw_channel_daily_total(i)
     dw_daily_channel()
     print('ok')
 
@@ -41,9 +43,11 @@ def run_cost_history():
         print(i)
         dw_daily_channel_cost(i)
         dw_order_channel_cost_sync_ck(i)
+        dw_channel_daily_total(i)
     dw_daily_channel()
 
 
+
 def main():
     run(yestoday)
     dw_daily_channel()

app/etl/data_stat_task.py

@@ -13,7 +13,7 @@ from sync_to_ck_task import dw_order_channel_cost_sync_ck
 def dw_daily_channel_cost(ymd):
     sql="""replace into dw_daily_channel_cost
         select dt,x.channel,pitcher,stage,platform,book,
-               ifnull(view_count,0),ifnull(click_count,0),ifnull(follow_user,0),ifnull(cost,0) as cost,
+               ifnull(view_count,0),ifnull(click_count,0),ifnull(follow_user,0),ifnull(cost,0)/100 as cost,
                ifnull(web_view_count,0) web_view_count,
                ifnull(platform_view_count,0) platform_view_count,
                ifnull(web_order_count,0) web_order_count
@@ -137,14 +137,17 @@ def order_account_text():
         for i in f.readlines():
             db.quchen_text.execute("insert into order_account_text(platform,text) values ('文鼎','{}')".format(i))
 
+
+
+
 def dw_daily_channel():
     """快照表 每日一更新  t-1"""
     sql="""insert into dw_daily_channel
 select
        dt,channel,pitcher,stage,platform,book,
        order_count,order_user,order_amount,first_order_count,first_order_user,first_order_amount,
-       view_count,click_count,follow_user,cost/100,reg_order_count,reg_order_user,reg_order_amount,reg_order_amount30,
-       web_view_count,platform_view_count,web_order_count,0 total_cost,0 total_amount
+       view_count,click_count,follow_user,cost,reg_order_count,reg_order_user,reg_order_amount,reg_order_amount30,
+       web_view_count,platform_view_count,web_order_count,total_cost,total_amount
 from
 (select dt,channel, pitcher,stage,platform,book,cost,view_count,click_count,follow_user,web_view_count,platform_view_count,web_order_count from dw_daily_channel_cost) a
     left outer join
@@ -152,7 +155,7 @@ from
        sum(amount) as reg_order_amount,
        count(distinct user_id) as reg_order_user,
        count(1) as reg_order_count,
-       sum(if(subtractDays(date, 30)>reg_time,toDecimal32(0,2),amount)) as reg_order_amount30
+       sum(if(subtractDays(date, 30)>reg_time,0,amount)) as reg_order_amount30
        from order where reg_time>'2019-03-18 00:00:00' group by toDate(formatDateTime(reg_time,'%Y-%m-%d')),channel) b
         on dt=dt2 and channel=channel2 left outer join
      (select date as dt3,channel as channel3,
@@ -167,7 +170,11 @@ from
         count(distinct user_id) as order_user,
              sum(amount) as order_amount
        from order  group by date,channel) d
-        on dt=dt4 and channel=channel4;"""
+        on dt=dt4 and channel=channel4
+left outer join
+     (select dt dt5,channel channel5,total_cost,total_amount from dw_channel_daily_total ) e
+         on dt=dt5 and channel=channel5   
+        """
     ck.execute("truncate table dw_daily_channel")
     ck.execute(sql)
     print("ok")
@@ -175,22 +182,23 @@ from
 
 def dm_pitcher_daily_page_total():
 
-    sql=f"""insert into dm_pitcher_daily_page_total
+    sql=f"""
+insert into dm_pitcher_daily_page_total
 select '{dt.get_n_days(-1)}' dt,
        pitcher,
-       cost,
-       amount,
-       roi,
+       round(cost,2),
+       round(amount,2),
+       round(roi,4),
        channel_count,
        on_channel_count,
        channel_count-on_channel_count off_channel_count,
-       this_month_cost,
-       this_month_amount,
-       this_month_roi,
-       last_month_cost,
-       last_month_amount,
-       last_month_roi,
-       last_month_far_amount,
+       round(this_month_cost,2),
+       round(this_month_amount,2),
+       round(this_month_roi,4),
+       round(last_month_cost,2),
+       round(last_month_amount,2),
+       round(last_month_roi,4),
+       round(last_month_far_amount,2),
        0 follow_user
 from (select pitcher,
              sum(cost)                                cost,
@@ -238,9 +246,21 @@ from (select pitcher,
     ) y on q.pitcher = y.pitcher
 having pitcher != ''"""
     print(sql)
+    ck.execute("alter table dm_pitcher_daily_page_total drop partition '{dt.get_n_days(-1)}';")
     ck.execute(sql)
 
 
+def dw_channel_daily_total(ymd):
+    """每天的公众号数据 （不包含向后数据）"""
+    sql=f"""insert into dw_channel_daily_total
+            select '{ymd}' dt,channel,total_cost,total_amount from
+            (select channel,sum(cost) total_cost  from dw_daily_channel_cost where  dt<='{ymd}'  group by channel)a
+            left outer join
+            (select  channel,sum(amount) total_amount  from order where date<='{ymd}' group by channel) b
+             on a.channel=b.channel"""
+    ck.execute(f"alter table dm_pitcher_daily_page_total drop partition '{ymd}'")
+    ck.execute(sql)
+
 if __name__ == '__main__':
     # channel_by_account_daily('2020-12-17')
     # dw_daily_channel_cost('2020-12-17')
@@ -252,13 +272,13 @@ if __name__ == '__main__':
     # dw_daily_channel()
     # exit(0)
     # dm_pitcher_daily_page_total()
+    # dw_channel_daily_total('2020-12-28')
     # exit(0)
 
-    for i in dt.getDateLists('2019-03-18','2020-12-21'):
+    for i in dt.getDateLists('2019-03-18','2021-01-04'):
         print(i)
-        ods_order(i)
+        # ods_order(i)
         # channel_info_daily(i)
         dw_daily_channel_cost(i)
         dw_order_channel_cost_sync_ck(i)
 
-