MOD:今日头条 channel change 添加

app/api_data/tx_ad_cost/get_cost_older.py

@@ -249,7 +249,7 @@ def get_daily_tt(st, et):
         access_token = refresh_access_token(appid, account_id, secret, refresh_token)
         # 3.获取agent_id
         sql = f'''
-        select distinct(advertiser_id) from bytedance_pitcher_change
+        select distinct(advertiser_id) from advertiser_bytedance
         where appid='{appid}' and account_id='{account_id}'
         '''
         advertiser_ids = db.quchen_text.getData(sql)

app/etl/data_stat_task.py

@@ -10,35 +10,58 @@ dt = DateUtils()
 def dw_daily_bytedance_cost(ymd):
     logging.info(f'dw_daily_bytedance_cost 数据填充开始')
 
-    dt_sql = f'''
-    select b.pitcher,b.channel ,a.`date` ,round(sum(cost)/100,2) as cost,sum(view_count) as view_count ,
-    sum(valid_click_count) as click_count ,c.stage as stage,d.book as book, e.current_platform as platform
-    from daily_tt a 
-    left join bytedance_pitcher_change b on a.account_id =b.advertiser_id 
-    left join stage_change c on b.channel =c.channel 
-    left join book_change d on b.channel =d.name 
-    left join platform_change e on b.channel = e.name
-    where a.`date`='{ymd}'
-    group by b.pitcher ,a.`date` ,b.channel 
-    '''
+    sql='''
+    select x.dt,x.channel,pitcher,stage,x.platform,x.book,
+               ifnull(view_count,0),ifnull(click_count,0),
+               ifnull(follow_user,0),ifnull(cost,0)/100 as cost,
+               ifnull(web_view_count,0) web_view_count,
+               ifnull(platform_view_count,0) platform_view_count,
+               ifnull(web_order_count,0) web_order_count,
+               'BYTEDANCE' type
+                ,0 require_roi,0 require_mult
+                 from
+        ( select dt, channel,stage,pitcher,platform,book  from channel_info_daily cid 
+            where dt='{0}' and channel !=''
+            and channel in 
+            (select distinct(channel) from channel_by_account_daily cbad 
+            where dt='{0}'
+            and type ='BYTEDANCE' 
+            )
+        ) x  -- 只允许渠道MP、GDT
+        left join
+        (select channel,sum(cost) as cost,sum(view_count) as view_count,sum(valid_click_count) as click_count,sum(from_follow_uv)  as follow_user,
+         sum(web_view_count) as web_view_count,sum(platform_view_count) as platform_view_count,sum(web_order_count) as web_order_count
+         from
+            (select account_id,cost,view_count,valid_click_count,round(valid_click_count*official_account_follow_rate,0) as from_follow_uv,
+            0 as  web_view_count,
+            0 as platform_view_count,
+            0 as web_order_count
+             from daily_tt where date='{0} 00:00:00' ) a
+        left join
+        (select account_id,channel from channel_by_account_daily where dt='{0}') b 
+         on a.account_id=b.account_id  group by channel) 
+        z on x.channel=z.channel  
+        
+        '''.format(ymd)
 
-    data_list = db.quchen_text.get_data_list(dt_sql)
-    byte_list = []
-    for _ in data_list:
-        _[2] = str(_[2]) if _[2] else 0
-        _[3] = round(float(_[3]), 2) if _[3] else 0
-        _[4] = round(float(_[4]), 2) if _[4] else 0
-        _[5] = round(float(_[5]), 2) if _[5] else 0
-        _[6] = str(_[6]) if _[6] else ''
-        _[7] = str(_[7]) if _[7] else ''
-        _[8] = str(_[8]) if _[8] else ''
-        _.append('BYTEDANCE')
-        byte_list.append(tuple(_))
-    col = '''pitcher,channel,dt,cost,view_count,click_count,stage,book,platform,type'''
-    logging.info(f'dw_daily_bytedance_cost add info {ymd}')
+    data = db.quchen_text.get_data_list(sql)
+    data1 = []
+    col = "dt,channel,pitcher,stage,platform,book,view_count,click_count,follow_user,cost,web_view_count,platform_view_count,web_order_count,type,require_roi,require_mult"
+    for i in data:
+        i[0] = str(i[0])
+        i[9] = str(i[9])
+        i[6] = float(i[6])
+        i[7] = float(i[7])
+        i[8] = float(i[8])
+        i[9] = float(i[9])
+        i[10] = float(i[10])
+        i[11] = float(i[11])
+        i[12] = float(i[12])
+        data1.append(tuple(i))
     ck.execute(f"alter table dw_daily_bytedance_cost drop  partition '{ymd}' ")
+    logging.info(len(data1))
+    ck.insertMany("dw_daily_bytedance_cost", col, tuple(data1))
 
-    ck.insertMany("dw_daily_bytedance_cost", col, tuple(byte_list))
     logging.info(f'dw_daily_bytedance_cost 数据填充结束')
 
 def platform_data_sum(ymd):
@@ -69,7 +92,14 @@ def dw_daily_channel_cost(ymd):
                if(stage ='趣程15期' or stage ='趣程26期' or stage ='趣程30期','GDT','MP') type
                 ,0 require_roi,0 require_mult
                  from
-        (select dt,channel,stage,pitcher,platform,book from channel_info_daily where dt='{0}' and channel!='' ) x  
+        ( select dt, channel,stage,pitcher,platform,book  from channel_info_daily cid 
+            where dt='{0}' and channel !=''
+            and channel in 
+            (select distinct(channel) from channel_by_account_daily cbad 
+            where dt='{0}'
+            and (type ='GDT' or type='MP')
+            ) 
+        ) x  -- 只允许渠道MP、GDT
         left join
         (select channel,sum(cost) as cost,sum(view_count) as view_count,sum(valid_click_count) as click_count,sum(from_follow_uv)  as follow_user,
          sum(web_view_count) as web_view_count,sum(platform_view_count) as platform_view_count,sum(web_order_count) as web_order_count
@@ -121,6 +151,8 @@ def channel_by_account_daily(ymd):
             (select  account_id,name,'GDT' type  from advertiser_qq 
              union 
              select account_id,name,'MP' type from advertiser_vx
+             union
+             select advertiser_id,channel,'BYTEDANCE' type from advertiser_bytedance
              ) a
                 left join
             (select b.account_id,b.name from
@@ -136,7 +168,6 @@ def channel_info_daily(ymd):
     """
     # 获取现在的全量公众号信息
     logging.info("run> channel_info_daily")
-
     sql = """select '{}' as dt,a.name ,ifnull(stage,''),ifnull(pitcher,''),ifnull(platform,''),ifnull(book,'') from (
             select  name from advertiser_vx  where name is not null group by name--  公众号全量表
             union
@@ -148,10 +179,23 @@ def channel_info_daily(ymd):
             union
             select  name from platform_change group by name
             union
-            select  name from book_change group by name) a left join (
-            select name,ifnull(stage,'') stage,ifnull(pitcher,'') pitcher,ifnull(platform,'') platform,ifnull(book,'') book from advertiser_qq where name is not null  group by name,stage,pitcher,platform,book
+            select  name from book_change group by name) a
+             
+             left join
+              (
+                select name,ifnull(stage,'') stage,ifnull(pitcher,'') pitcher,
+                ifnull(platform,'') platform,ifnull(book,'') book 
+                from advertiser_qq where name is not null  
+                group by name,stage,pitcher,platform,book
              union
-                select name,ifnull(stage,'') stage,ifnull(pitcher,'') pitcher,ifnull(platform,'') platform,ifnull(book,'') book from advertiser_vx where name is not null and name !=''
+                select name,ifnull(stage,'') stage,ifnull(pitcher,'') pitcher,
+                ifnull(platform,'') platform,ifnull(book,'') book 
+                from advertiser_vx where name is not null and name !=''
+            union
+                select channel,ifnull(stage,'') stage,ifnull(pitcher,'') pitcher,
+                ifnull(platform,'') platform,ifnull(book,'') book
+                from advertiser_bytedance where channel is not null and channel !=''
+                group by channel ,stage,pitcher,platform,book
                 ) b on a.name=b.name
                 """.format(ymd)
     data = db.quchen_text.get_data_list(sql)