1

app/etl/data_stat_daily.py

@@ -19,8 +19,8 @@ def run(ymd):
         order_sync_ck(ymd)
 
         # 公众号维度数据-同步到ck
-        dw_daily_channel(ymd)
-        dw_order_channel_sync_ck(ymd)
+        dw_daily_channel_cost(ymd)
+        dw_order_channel_cost_sync_ck(ymd)
     except Exception as e:
         DingTalkUtils.send("数据ETL出错：",'18860455786')
 

app/etl/data_stat_task.py

@@ -6,25 +6,32 @@ db = MysqlUtils()
 ck = CkUtils()
 dt = DateUtils()
 from datetime import datetime
-from sync_to_ck_task import dw_order_channel_sync_ck
-
-def dw_daily_channel(ymd):
-    sql="""replace into dw_daily_channel
-        select dt,x.channel,pitcher,stage,platform,book,ifnull(order_count,0) order_count,ifnull(order_user,0),ifnull(order_amount,0),ifnull(first_order_count,0),ifnull(first_order_user,0),
-               ifnull(first_order_amount,0),ifnull(view_count,0),ifnull(click_count,0),ifnull(follow_user,0),ifnull(cost,0) as cost,type   from
-        (select dt,channel,stage,pitcher,platform,book from channel_info_daily where dt='{0}' and channel!='') x
-        left join
-        (select  channel,count(1) as order_count,count(distinct user_id) as order_user,sum(amount) as order_amount,
-               sum(if(date_format(reg_time,'%Y-%m-%d')=date_format(date,'%Y-%m-%d'),1,0)) as first_order_count,
-               count(distinct if(date_format(reg_time,'%Y-%m-%d')=date_format(date,'%Y-%m-%d'),user_id,''))-1 as first_order_user,
-               sum(if(date_format(reg_time,'%Y-%m-%d')=date_format(date,'%Y-%m-%d'),amount,0)) as first_order_amount
-        from ods_order where date='{0}' group by channel) y on x.channel=y.channel
-        
+from sync_to_ck_task import dw_order_channel_cost_sync_ck
+
+def dw_daily_channel_cost(ymd):
+    sql="""replace into dw_daily_channel_cost
+        select dt,x.channel,pitcher,stage,platform,book,
+               ifnull(view_count,0),ifnull(click_count,0),ifnull(follow_user,0),ifnull(cost,0) as cost,
+               ifnull(web_view_count,0) web_view_count,
+               ifnull(platform_view_count,0) platform_view_count,
+               ifnull(web_order_count,0) web_order_count
+                 from
+        (select dt,channel,stage,pitcher,platform,book from channel_info_daily where dt='{0}' and channel!='') x  
         left join
-        (select channel,type,sum(cost) as cost,sum(view_count) as view_count,sum(valid_click_count) as click_count,sum(from_follow_uv) as follow_user from
-        (select account_id,cost,view_count,valid_click_count,round(valid_click_count*official_account_follow_rate,0) as from_follow_uv from daily_vx where date='{0} 00:00:00'
-        union
-        select account_id,cost,view_count,valid_click_count,from_follow_uv from daily_qq where date='{0} 00:00:00') a
+        (select channel,type,sum(cost) as cost,sum(view_count) as view_count,sum(valid_click_count) as click_count,sum(from_follow_uv)  as follow_user,
+         sum(web_view_count) as web_view_count,sum(platform_view_count) as platform_view_count,sum(web_order_count) as web_order_count
+         from
+            (select account_id,cost,view_count,valid_click_count,round(valid_click_count*official_account_follow_rate,0) as from_follow_uv,
+            0 as  web_view_count,
+            0 as platform_view_count,
+            0 as web_order_count
+             from daily_vx where date='{0} 00:00:00'
+            union
+            select account_id,cost,view_count,valid_click_count,from_follow_uv,
+            ifnull(web_commodity_page_view_count,0) as  web_view_count,
+            ifnull(platform_page_view_count,0) as platform_view_count,
+            ifnull(web_order_count,0) as web_order_count
+             from daily_qq where date='{0} 00:00:00') a
         left join
         (select account_id,channel,type from channel_by_account_daily where dt='{0}') b on a.account_id=b.account_id group by channel,type) z on x.channel=z.channel 
         """.format(ymd)
@@ -159,17 +166,19 @@ def dw_daily_channel_plus():
 
 if __name__ == '__main__':
     # channel_by_account_daily('2020-12-17')
-
-    ods_order('2020-12-20')
+    # dw_daily_channel_cost('2020-12-17')
+    # dw_order_channel_cost_sync_ck('2020-12-17')
+    # exit(0)
+    # ods_order('2020-12-20')
     # dw_daily_channel_plus()
-    exit()
+    # exit()
 
-    # for i in dt.getDateLists('2019-03-18','2020-12-17'):
-    #     print(i)
+    for i in dt.getDateLists('2019-03-18','2020-12-20'):
+        print(i)
         # ods_order(i)
         # channel_by_account_daily(i)
         # channel_info_daily(i)
-        # dw_daily_channel(i)
-        # dw_order_channel_sync_ck(i)
+        # dw_daily_channel_cost(i)
+        dw_order_channel_cost_sync_ck(i)
 
 

app/etl/sync_to_ck_task.py

@@ -65,18 +65,16 @@ def campaign_vx():
     # data=db.quchen_text.getData(sql)
     pass
 
-def dw_order_channel_sync_ck(ymd):
-    table='dw_daily_channel'
+def dw_order_channel_cost_sync_ck(ymd):
+    table='dw_daily_channel_cost'
     sql = f"select * from {table} where dt='{ymd}'"
     col=ck.getColumns(f"{table}",str=True)
     data = db.quchen_text.get_data_list(sql)
     data1=[]
     for i in data:
         i[0]=str(i[0])
-        i[8]=str(i[8])
-        i[11]=str(i[11])
-        i[15]=str(i[15])
-        i[-1]='' if i[-1]==None else i[-1]
+        i[9]=str(i[9])
+
         data1.append(tuple(i))
     ck.execute(f"alter table {table} drop  partition '{ymd}' ")
     ck.insertMany(f"{table}", col, tuple(data1))
@@ -85,7 +83,8 @@ def dw_order_channel_sync_ck(ymd):
 
 
 if __name__ == '__main__':
-    dw_order_channel_sync_ck(dt.get_n_days(-2))
+    order_sync_ck('2020-12-20')
+    dw_order_channel_cost_sync_ck(dt.get_n_days(-2))
     # for i in dt.getDateLists('2019-03-18','2020-12-17'):
     # #     order(i)
     #     dw_order_channel_sync_ck(i)