MOD:添加多渠道数据处理

app/crontab_task/task.py

@@ -90,7 +90,7 @@ def daily():
     t11.start()
     t11.join()
 
-    yangguang.get_channel_info()
+    # yangguang.get_channel_info()
 
 
 def cost_yestoday_repair():

app/etl/data_stat_run.py

@@ -35,6 +35,8 @@ def do_cost(st, et):
         channel_by_account_daily(i)
         channel_info_daily(i)
         dw_daily_channel_cost(i)
+        dw_daily_bytedance_cost(i)
+        platform_data_sum(i)
     logging.info('消耗数据处理,结束')
 
 

app/etl/data_stat_task.py

@@ -7,6 +7,56 @@ ck = CkUtils()
 dt = DateUtils()
 
 
+def dw_daily_bytedance_cost(ymd):
+    logging.info(f'dw_daily_bytedance_cost 数据填充开始')
+
+    dt_sql = f'''
+    select b.pitcher,b.channel ,a.`date` ,round(sum(cost)/100,2) as cost,sum(view_count) as view_count ,
+    sum(valid_click_count) as click_count ,c.stage as stage,d.book as book, e.current_platform as platform
+    from daily_tt a 
+    left join bytedance_pitcher_change b on a.account_id =b.advertiser_id 
+    left join stage_change c on b.channel =c.channel 
+    left join book_change d on b.channel =d.name 
+    left join platform_change e on b.channel = e.name
+    where a.`date`='{ymd}'
+    group by b.pitcher ,a.`date` ,b.channel 
+    '''
+
+    data_list = db.quchen_text.get_data_list(dt_sql)
+    byte_list = []
+    for _ in data_list:
+        _[2] = str(_[2]) if _[2] else 0
+        _[3] = round(float(_[3]), 2) if _[3] else 0
+        _[4] = round(float(_[4]), 2) if _[4] else 0
+        _[5] = round(float(_[5]), 2) if _[5] else 0
+        _[6] = str(_[6]) if _[6] else ''
+        _[7] = str(_[7]) if _[7] else ''
+        _[8] = str(_[8]) if _[8] else ''
+        _.append('BYTEDANCE')
+        byte_list.append(tuple(_))
+    col = '''pitcher,channel,dt,cost,view_count,click_count,stage,book,platform,type'''
+    logging.info(f'dw_daily_bytedance_cost add info {ymd}')
+    ck.execute(f"alter table dw_daily_bytedance_cost drop  partition '{ymd}' ")
+
+    ck.insertMany("dw_daily_bytedance_cost", col, tuple(byte_list))
+    logging.info(f'dw_daily_bytedance_cost 数据填充结束')
+
+def platform_data_sum(ymd):
+    logging.info('dw_daily_platform_cost开始数据更新')
+    ck.execute("alter table dw_daily_platform_cost drop  partition '{}' ".format(ymd))
+    sql=f'''
+        insert into dw_daily_platform_cost
+        select * from 
+        (select * from dw_daily_bytedance_cost a where dt='{ymd}'
+        union all 
+        select * from dw_daily_channel_cost b where dt='{ymd}'
+        AND channel not  in (select channel from dw_daily_bytedance_cost a 
+        where dt='{ymd}'))
+    '''
+    ck.execute(sql)
+    logging.info('dw_daily_platform_cost数据更新，结束')
+
+
 def dw_daily_channel_cost(ymd):
     logging.info("run> dw_daily_channel_cost")
     sql = """
@@ -175,7 +225,7 @@ if __name__ == '__main__':
     # channel_info_daily('2021-02-06')
 
     # channel_by_account_daily('2021-02-05')
-    # for i in dt.getDateLists('2020-04-23','2021-02-05'):
-    #     print(i)
-    #     dw_daily_channel_cost(i)
-    ods_order('2021-05-06')
+    for i in dt.getDateLists('2021-07-23', '2021-09-17'):
+        print(i)
+        dw_daily_channel_cost(i)
+    # ods_order('2021-05-06')

app/etl/dw/dw_channel_daily.py

@@ -15,7 +15,8 @@ ck = CkUtils()
 def dw_channel():
     sql = """
 select
-       dt1,channel1,pitcher,stage,platform,book,if(stage ='趣程15期' or stage ='趣程26期' or stage ='趣程30期','GDT','MP') type,
+       dt1,channel1,pitcher,stage,platform,book,
+       if(type not in ('BYTEDANCE'),if(stage ='趣程15期' or stage ='趣程26期' or stage ='趣程30期','GDT','MP'),type )type,
        order_count,order_user,order_amount,
        first_order_count,first_order_user,first_order_amount,
        view_count,click_count,follow_user,
@@ -33,7 +34,7 @@ from (
         follow_user,web_view_count,platform_view_count,web_order_count,type,
         require_roi,require_mult,order_count,order_user,order_amount from
 
-        (select * from dw_daily_channel_cost) aa
+        (select * from dw_daily_platform_cost) aa
         full outer  join
         (select date as dt4,channel as channel4,count(1) as order_count,   ---账面充值
         count(distinct user_id) as order_user,sum(amount) as order_amount