1

app/etl/data_stat_daily.py

@@ -18,6 +18,7 @@ def _channel_info_daily():
 if __name__ == '__main__':
     _channel_by_account_daily()
     _channel_info_daily()
+    dw_daily_channel(yestoday)
 
 
 

app/etl/data_stat_hourly.py

@@ -3,11 +3,11 @@ from model.log import logger
 from model.DingTalkUtils import DingTalkUtils
 from data_stat_task import *
 log=logger()
-dt = DateUtils()
-
+du = DateUtils()
+yestoday = du.get_n_days(-1)
+today = du.get_n_days(0)
 
 if __name__ == '__main__':
-    today = dt.get_n_days()
     dw_daily_channel(today)
 
 

app/etl/data_stat_task.py

@@ -8,47 +8,26 @@ dt = DateUtils()
 from datetime import datetime
 
 
-def dw_daily_channel(dt):
-    sql = """select '{0}' as dt,c.channel as channel,pitcher,stage,order_count,order_user,order_amount,first_order_count,first_order_user,first_order_amount,
-          view_count,click_count, cost
-
-           from
-    (select channel,order_count,order_user,order_amount,first_order_count,first_order_user,first_order_amount from
-    (select channel,count(1) as order_count,count(distinct user_id) as order_user,sum(amount) as order_amount from order
-    where date='{0}'
-    group by channel) a
-        left outer JOIN
-
-    (select channel,count(1) as first_order_count,
-    count(distinct user_id) as first_order_user,
-    sum(amount) as first_order_amount from order
-     where formatDateTime(reg_time,'%Y-%m-%d')='{0}' and date ='{0}'
-     group by channel )  b
-        on a.channel=b.channel) p
-
-      right OUTER JOIN
-
-    ( select channel,pitcher,stage,sum(cost) as cost, sum(view_count) as view_count,sum(valid_click_count) as click_count from
-    (select account_id,cost,view_count,valid_click_count  from daily_qq where date='{0}'
-     union all
-     select account_id,cost,view_count,valid_click_count from daily_vx where date='{0}') h
-    right outer join
-    (select account_id,name as channel,stage,pitcher from advertiser_vx_qq where name!='') j on h.account_id=j.account_id
-        group by channel,stage,pitcher) c
-        on p.channel=c.channel""".format(dt)
-
-    data = ck.execute(sql)
-
-    data1 = []
-    for i in data:
-        li = list(i)
-        li[6] = int(li[6])
-        li[9] = int(li[9])
-        data1.append(tuple(li))
-
-    insert_sql = "insert into dw_daily_channel values {}".format(str(data1)[1:-1])
-    # print(insert_sql)
-    db.quchen_text.execute(insert_sql)
+def dw_daily_channel(ymd):
+    sql="""replace into dw_daily_channel
+        select dt,x.channel,pitcher,stage,platform,book,ifnull(order_count,0),ifnull(order_user,0),ifnull(order_amount,0),ifnull(first_order_count,0),ifnull(first_order_user,0),
+               ifnull(first_order_amount,0),ifnull(view_count,0),ifnull(click_count,0),ifnull(follow_user,0),ifnull(cost,0) from
+        (select dt,channel,stage,pitcher,platform,book from channel_info_daily where dt='{0}' and channel!='') x
+        left join
+        (select  channel,count(1) as order_count,count(distinct user_id) as order_user,sum(amount) as order_amount,
+               sum(if(date_format(reg_time,'%Y-%m-%d')=date_format(date,'%Y-%m-%d'),1,0)) as first_order_count,
+               count(distinct if(date_format(reg_time,'%Y-%m-%d')=date_format(date,'%Y-%m-%d'),user_id,''))-1 as first_order_user,
+               sum(if(date_format(reg_time,'%Y-%m-%d')=date_format(date,'%Y-%m-%d'),amount,0)) as first_order_amount
+        from ods_order where date='{0}' group by channel) y on x.channel=y.channel
+        
+        left join
+        (select channel,sum(cost) as cost,sum(view_count) as view_count,sum(valid_click_count) as click_count,sum(from_follow_uv) as follow_user from
+        (select account_id,cost,view_count,valid_click_count,round(valid_click_count*official_account_follow_rate,0) as from_follow_uv from daily_vx where date='{0} 00:00:00'
+        union
+        select account_id,cost,view_count,valid_click_count,from_follow_uv from daily_qq where date='{0} 00:00:00') a
+        left join
+        (select account_id,channel from channel_by_account_daily where dt='{0}') b on a.account_id=b.account_id group by channel) z on x.channel=z.channel""".format(ymd)
+    db.quchen_text.execute(sql)
 
 
 
@@ -56,7 +35,10 @@ def channel_by_account_daily(ymd):
     """返回当天消耗账户对应的公众号表"""
     sql="""replace into channel_by_account_daily 
             select  '{0}' as dt,a.account_id as account_id, ifnull(ifnull(b.name,a.name),'') as channel  from
-            (select  account_id,name from advertiser_qq) a
+            (select  account_id,name from advertiser_qq 
+             union 
+             select account_id,name from advertiser_vx
+             ) a
                 left join
             (select b.account_id,b.name from
             (select min(end_time) as end_time,account_id from account_change where end_time>'{0}' GROUP BY account_id) a
@@ -82,7 +64,7 @@ def channel_info_daily(ymd):
             select  name from book_change group by name) a left join (
             select name,ifnull(stage,'') stage,ifnull(pitcher,'') pitcher,ifnull(platform,'') platform,ifnull(book,'') book from advertiser_qq where name is not null  group by name,stage,pitcher,platform,book
              union
-                select name,ifnull(stage,'') stage,ifnull(pitcher,'') pitcher,ifnull(platform,'') platform,ifnull(book,'') book from advertiser_vx where name is not null
+                select name,ifnull(stage,'') stage,ifnull(pitcher,'') pitcher,ifnull(platform,'') platform,ifnull(book,'') book from advertiser_vx where name is not null and name !=''
                 ) b on a.name=b.name""".format(ymd)
     data=db.quchen_text.get_data_list(sql)
 
@@ -151,8 +133,10 @@ def order_account_text():
 
 
 if __name__ == '__main__':
+    # dw_daily_channel('2020-12-14')
     # channel_info_daily('2020-12-14')
     for i in dt.getDateLists('2019-03-18','2020-12-16'):
         print(i)
-        channel_info_daily(i)
-
+        # channel_by_account_daily(i)
+        # channel_info_daily(i)
+        dw_daily_channel(i)