ADD：数据变为两sdk合并

app/api_data/platform_order/order_data_change.py

@@ -6,10 +6,11 @@ import logging
 
 du = DateUtils()
 db = MysqlUtils()
+ck = CkUtils()
 
 
-def insert_order_data(ymd):
-    #TODO: channel_id 有问题，不可以使用app_id游戏id
+def insert_order_data_game_one(ymd):
+    # TODO: channel_id 有问题，不可以使用app_id游戏id
     logging.info('订单数据格式转换 ，开始 ')
     datatime_ymd = datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))).timetuple()
     timestamp_ymd = time.mktime(datatime_ymd)
@@ -18,7 +19,8 @@ def insert_order_data(ymd):
     timestamp_tom = time.mktime(tomorrow_ymd)
 
     sql = f'''
-    select DATE(FROM_UNIXTIME(a.create_time)),'',payway as order_type,c.wx_name,a.app_id ,f.id as user_id,
+    select DATE(FROM_UNIXTIME(a.create_time)),'',payway as order_type,c.wx_name,a.app_id ,
+    concat('sdk_one_',f.id) as user_id,
     DATE_FORMAT(FROM_UNIXTIME(a.create_time), '%Y-%m-%d %H:%i:%s') as order_time,
     DATE_FORMAT(FROM_UNIXTIME(f.create_time), '%Y-%m-%d %H:%i:%s') as reg_time,
     a.amount ,ifnull(d.name,''),a.order_id ,a.status 
@@ -31,7 +33,7 @@ def insert_order_data(ymd):
         left join db_mp.mp_conf_agent b on  a.agent_id=b.agent_id 
         and a.update_time = b.update_time ) b on  a.agent_id =b.agent_id 
     left join db_mp.mp_mp_conf c on b.advertiser_conf_id = c.id 
-    left join db_mp.h_game d on a.app_id =d.id 
+    left join db_mp.h_game d on a.app_id = d.id 
     left join db_mp.h_member f on a.mem_id = f.id
     where c.`type` in (6,7)
     and a.create_time<={timestamp_tom} and a.create_time>={timestamp_ymd}
@@ -50,15 +52,100 @@ def insert_order_data(ymd):
     logging.info('订单数据格式转换 ，结束 ')
 
 
+def insert_order_data_ck():
+    logging.info('订单数据格式转换 ，开始 ')
+
+    sql = f'''
+        insert into game_data.order
+        (date,stage,platform,channel,channel_id,user_id,
+                order_time,
+                reg_time,
+                amount,from_novel,order_id,status)
+        select toDate(FROM_UNIXTIME(a.create_time)),'',
+        payway as order_type,
+        c.wx_name,a.app_id ,f.id as user_id,
+        FROM_UNIXTIME(a.create_time) as order_time,
+        FROM_UNIXTIME(f.create_time) as reg_time,
+        a.amount ,ifnull(d.name,''),a.order_id ,a.status 
+        from GameDataSum.h_pay_sum a 
+        left join 
+        (select b.agent_id agent_id,b.advertiser_conf_id advertiser_conf_id  from 
+            (select agent_id,max(update_time) as update_time  
+            from GameDataSum.mp_conf_agent mmc 
+            group by agent_id ) a
+            left join GameDataSum.mp_conf_agent b on  a.agent_id=b.agent_id 
+            and a.update_time = b.update_time ) b on  a.agent_id =b.agent_id 
+        left join GameDataSum.mp_mp_conf c on b.advertiser_conf_id = c.id 
+        left join (select toUInt32(id) id,name from GameDataSum.h_game) d
+                on a.app_id = d.id
+        left join (select toUInt32(id) id,create_time from GameDataSum.h_member) f
+        on a.mem_id = f.id
+        where c.`type` in (6,7)
+        '''
+    print(sql)
+    ck.execute(sql)
+
+    final_sql = '''OPTIMIZE table game_data.order final '''
+    ck.execute(final_sql)
+
+    logging.info('订单数据格式转换 ，结束 ')
+
+
+def insert_order_data_ck_gamedatasum(ymd):
+    logging.info('订单数据格式转换 ，开始 ')
+    datatime_ymd = datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))).timetuple()
+    timestamp_ymd = time.mktime(datatime_ymd)
+    tomorrow_ymd = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=1)).timetuple()
+    timestamp_tom = time.mktime(tomorrow_ymd)
+
+    sql = f'''
+        insert into game_data.order
+        (date,stage,platform,channel,channel_id,user_id,
+                order_time,
+                reg_time,
+                amount,from_novel,order_id,status)
+        select toDate(FROM_UNIXTIME(a.create_time)),'',
+        payway as order_type,
+        c.wx_name,a.app_id ,f.id as user_id,
+        FROM_UNIXTIME(a.create_time) as order_time,
+        FROM_UNIXTIME(f.create_time) as reg_time,
+        a.amount ,ifnull(d.name,''),a.order_id ,a.status 
+       from GameDataSum.h_pay_sum a 
+        left join 
+        (select b.agent_id agent_id,b.advertiser_conf_id advertiser_conf_id  from 
+            (select agent_id,max(update_time) as update_time  
+            from GameDataSum.mp_conf_agent_sum mmc 
+            group by agent_id ) a
+            left join GameDataSum.mp_conf_agent_sum b on  a.agent_id=b.agent_id 
+            and a.update_time = b.update_time ) b on  a.agent_id =b.agent_id 
+        left join GameDataSum.mp_mp_conf_sum c on b.advertiser_conf_id = c.id 
+        left join GameDataSum.h_game_sum d on a.app_id = d.id
+        left join  GameDataSum.h_member_sum f on a.mem_id = f.id
+        where c.`type` in (6,7)
+            and a.create_time<={timestamp_tom} and a.create_time>={timestamp_ymd}
+
+        '''
+    print(sql)
+    ck.execute(sql)
+
+    final_sql = '''OPTIMIZE table game_data.order final '''
+    ck.execute(final_sql)
+
+    logging.info('订单数据格式转换 ，结束 ')
+
+
 def insert_order_data_hourly():
-    insert_order_data(du.getNow())
+    insert_order_data_ck_gamedatasum(du.getNow())
 
 
 def insert_order_data_daily():
     for i in du.getDateLists(du.get_n_days(-10), du.get_n_days(0)):
         print(i)
-        insert_order_data(i)
+        insert_order_data_ck_gamedatasum(i)
 
 
 if __name__ == '__main__':
-    insert_order_data_daily()
+    # insert_order_data_daily()
+    # insert_order_data_ck()
+    insert_order_data_hourly()

app/crontab_task/task.py

@@ -3,31 +3,41 @@ from app.api_data.tx_ad_cost.get_cost_older import old_cost_hourly, old_cost_dai
 from app.etl.sync_to_ck_task import order_sync_ck
 from app.api_data.tx_ad_cost import get_cost_older
 from app.etl.data_stat_run import do_cost
-from app.api_data.platform_order.order_data_change import insert_order_data_daily,insert_order_data_hourly
+from app.api_data.platform_order.order_data_change import insert_order_data_daily, insert_order_data_hourly
 from model.DateUtils import DateUtils
+from app.game_data_sum.data_sum import change_data_style_daily, change_data_style_hourly
 
 du = DateUtils()
 
+
 def hourly():
+    t10 = threading.Thread(target=change_data_style_hourly)
     t11 = threading.Thread(target=old_cost_hourly)
-    t12 = threading.Thread(target=insert_order_data_hourly)
+    # t12 = threading.Thread(target=insert_order_data_hourly)
+
+    t10.start()
+    t10.join()
 
     t11.start()
     t11.join()
 
-    t12.start()
-    t12.join()
+    # t12.start()
+    # t12.join()
 
-def daily():
 
+def daily():
+    t10 = threading.Thread(target=change_data_style_daily)
     t11 = threading.Thread(target=old_cost_daily, args=())
-    t12 = threading.Thread(target=insert_order_data_daily)
+    # t12 = threading.Thread(target=insert_order_data_daily)
+
+    t10.start()
+    t10.join()
 
     t11.start()
     t11.join()
 
-    t12.start()
-    t12.join()
+    # t12.start()
+    # t12.join()
 
 
 def cost_yestoday_repair():
@@ -36,11 +46,10 @@ def cost_yestoday_repair():
     do_cost(dt, dt)
 
 
-
-
 if __name__ == '__main__':
     import logging
     from logging import handlers
+
     logging.basicConfig(
         handlers=[
             logging.handlers.RotatingFileHandler('task.log',
@@ -54,4 +63,4 @@ if __name__ == '__main__':
     )
     # hourly()
     # cost_yestoday_repair()
-    daily()
+    daily()

app/etl/data_stat_run.py

@@ -10,6 +10,7 @@ from app.etl.src.src_book_info import src_book_info
 from app.etl.dw.dw_book_trend import book_trend
 from app.etl.src import book_annual_expect_profit
 from logging import handlers
+from app.api_data.platform_order.order_data_change import insert_order_data_ck_gamedatasum
 
 du = DateUtils()
 logger = logging.getLogger("")
@@ -23,18 +24,17 @@ def do_order(st, et):
     logging.info('订单数据同步到ck,开始')
     for i in du.getDateLists(st, et):
         logging.info('订单:' + str(i))
-        order_sync_ck(i)
+        insert_order_data_ck_gamedatasum(i)
     logging.info('订单数据同步到ck,结束')
 
 
-
 def do_cost(st, et):
     logging.info('消耗数据处理,开始')
     for i in du.getDateLists(st, et):
         logging.info("消耗:" + str(i))
         channel_by_account_daily(i)
         channel_info_daily(i)
-        dw_daily_channel_cost(i)
+        dw_daily_channel_cost_ck_gamedatasum(i)
         # dw_daily_bytedance_cost(i)
         platform_data_sum(i)
     logging.info('消耗数据处理,结束')
@@ -44,8 +44,6 @@ def main(st, et):
     try:
         do_order(st, et)
         do_cost(st, et)
-        # src_book_info()  # 书籍卡点信息
-        # book_annual_expect_profit.run() # 年预期收益
         dw_channel_daily()
         dw_pitcher_trend()
         book_trend()
@@ -57,13 +55,12 @@ def main(st, et):
 
 def hourly():
     thedate = du.getTodayOrYestoday()
-    #默认做三天的留存数据
-    for i in du.getDateLists(du.get_n_days(-3), du.getNow()):
-        dw_daily_channel_cost(i)
+    # 默认做三天的留存数据
+    # for i in du.getDateLists(du.get_n_days(-15), du.getNow()):
+    #     dw_daily_channel_cost(i)
     main(thedate, thedate)
 
 
-
 def daily():
     "往前跑10天"
     st = du.get_n_days(-10)
@@ -72,12 +69,10 @@ def daily():
     do_cost(st, et)
 
 
-
-
 if __name__ == '__main__':
     logging.basicConfig(
         handlers=[
-            logging.handlers.RotatingFileHandler('./log/data_stat_run.log',
+            logging.handlers.RotatingFileHandler('./data_stat_run.log',
                                                  maxBytes=10 * 1024 * 1024,
                                                  backupCount=5,
                                                  encoding='utf-8')
@@ -86,7 +81,8 @@ if __name__ == '__main__':
         level=logging.INFO,
         format="%(asctime)s - %(levelname)s %(filename)s %(funcName)s %(lineno)s - %(message)s"
     )
-    main(du.get_n_days(-200), du.getNow())
+    do_cost('2021-09-01','2022-04-20')
+    # main(du.get_n_days(-200), du.getNow())
     # hourly()
     # daily()
     # daily('2021-06-30','2021-07-04')

app/etl/data_stat_task.py

@@ -20,6 +20,651 @@ def platform_data_sum(ymd):
     logging.info('dw_daily_platform_cost数据更新，结束')
 
 
+def dw_daily_channel_cost_ck_gamedatasum(ymd):
+    logging.info("run> dw_daily_channel_cost")
+    datatime_ymd = datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))).timetuple()
+    datatime_ymd_tom = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=1)).timetuple()
+
+    # datatime_str
+    ymd_tom = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=1)).strftime('%Y-%m-%d')
+    ymd_tom_after = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=2)).strftime('%Y-%m-%d')
+    ymd_seven_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=7)).strftime('%Y-%m-%d')
+    ymd_fifteen_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=15)).strftime('%Y-%m-%d')
+    ymd_thirty_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=30)).strftime('%Y-%m-%d')
+    ymd_fortyfive_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=45)).strftime('%Y-%m-%d')
+    ymd_sixty_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=60)).strftime('%Y-%m-%d')
+
+    # timestamp
+    timestamp_ymd = time.mktime(datatime_ymd)
+    timestamp_tom = time.mktime(datatime_ymd_tom)
+
+    ck.execute(f"alter table game_data.dw_daily_channel_cost drop  partition '{ymd}' ")
+
+    # TODO:创建角色量有问题
+    ck_sql = f'''
+    insert into game_data.dw_daily_channel_cost 
+        (dt,channel,pitcher,stage,platform,book,view_count,click_count,follow_user,cost,
+        web_view_count,platform_view_count,web_order_count,type,require_roi,require_mult,
+        reg_num,create_user_num,today_active_user_rate,second_stay_rate,third_stay_rate,
+        seven_stay_rate,fifteen_stay_rate,thirty_stay_rate,fortyfive_stay_rate,
+        sixty_stay_rate,game_user_sum)
+    select x.dt,x.channel,pitcher,stage,x.platform,x.book,
+               ifnull(view_count,0),ifnull(click_count,0),
+               ifnull(follow_user,0),ifnull(cost,0)/100 as cost,
+               ifnull(web_view_count,0) web_view_count,
+               ifnull(platform_view_count,0) platform_view_count,
+               ifnull(web_order_count,0) web_order_count,
+               if(stage ='趣程15期' or stage ='趣程26期' or stage ='趣程30期','GDT','MP') type
+                ,0 require_roi,0 require_mult,
+                ifnull(y.reg_num,0),ifnull(w.create_user_num,0),
+                v.today_active_user_rate,
+                v.second_stay_rate,
+                v.third_stay_rate,
+                v.seven_stay_rate, 
+                v.fifteen_stay_rate,
+                v.thirty_stay_rate,
+                v.fortyfive_stay_rate,
+                v.sixty_stay_rate,
+                v.game_user_sum
+        from
+        ( select dt, channel,stage,pitcher,platform,book  from CostSourceData.channel_info_daily final 
+            where dt='{ymd}' and channel !=''
+            and channel in 
+            (select distinct(channel) from CostSourceData.channel_by_account_daily final 
+            where dt='{ymd}'
+            and (type ='GDT' or type='MP')
+            ) 
+        ) x  -- 只允许渠道MP、GDT
+        left join
+        
+        ( select channel,
+		sum(cost) as cost,
+		sum(view_count) as view_count,
+		sum(valid_click_count) as click_count,
+          sum(from_follow_uv)  as follow_user,
+         sum(web_view_count) as web_view_count,
+         sum(platform_view_count) as platform_view_count,
+         sum(web_order_count) as web_order_count
+         from
+            (select account_id,cost,view_count,valid_click_count,
+            round(valid_click_count*official_account_follow_rate,0) as from_follow_uv,
+            0 as  web_view_count,
+            0 as platform_view_count,
+            0 as web_order_count
+             from CostSourceData.daily_vx final
+             where date=toDateTime('{ymd} 00:00:00')
+            union all
+            select account_id,cost,view_count,valid_click_count,from_follow_uv,
+            ifnull(web_commodity_page_view_count,0) as  web_view_count, 
+            ifnull(platform_page_view_count,0) as platform_view_count,
+            ifnull(web_order_count,0) as web_order_count
+             from CostSourceData.daily_qq final
+             where date=toDateTime('{ymd} 00:00:00')
+             ) a
+        left join
+        (select toString(account_id) account_id,channel 
+        from CostSourceData.channel_by_account_daily final 
+        			where dt=toDate('{ymd}')) b 
+         on a.account_id=b.account_id  group by channel) 
+        z on x.channel=z.channel   -- 只允许渠道cost  消耗
+  
+  
+   left join     
+        (
+        select channel,wx_date,count(*) reg_num
+        from  (
+         select h.name as channel ,toDate(create_time) wx_date
+        from 
+        GameDataSum.h_member_sum origin left join
+        (select a.name name,d.app_id app_id ,d.agent_id agent_id 
+            from ( select * from CostSourceData.advertiser_vx final) a
+			left join GameDataSum.h_game_sum b on a.book = b.name 
+			left join GameDataSum.mp_mp_conf_sum c on a.name= c.wx_name 
+			left join GameDataSum.mp_conf_agent_sum d on c.id=d.advertiser_conf_id 
+			where d.app_id =b.id
+			group by a.name,d.app_id ,d.agent_id ) h 
+			on origin.app_id = h.app_id and origin.agent_id = h.agent_id
+            where h.name is not null      
+            and origin.create_time > {timestamp_ymd} and origin.create_time < {timestamp_tom}  
+        ) tmp
+        group by channel, wx_date
+        ) y on x.channel= y.channel  
+        --- 注册用户数
+  
+  
+   left join     
+        (
+        select  channel, wx_date,
+        count(*) as create_user_num from 
+        (select h.name as channel,DATE(FROM_UNIXTIME(c.create_time)) as wx_date
+        from ( select * from GameDataSum.h_mg_role_sum where  create_time >= {timestamp_ymd} ) a
+        left join GameDataSum.h_mem_game_sum b on a.mg_mem_id = b.id
+        left join (select * from GameDataSum.h_member_sum                 
+                    where create_time >= {timestamp_ymd} and create_time <= {timestamp_tom} ) c 
+            on b.mem_id = c.id
+        left join
+        (select a.name name ,d.app_id app_id ,d.agent_id agent_id 
+            from ( select * from CostSourceData.advertiser_vx final) a
+			left join GameDataSum.h_game_sum b on a.book = b.name 
+			left join GameDataSum.mp_mp_conf_sum c on a.name= c.wx_name 
+			left join GameDataSum.mp_conf_agent_sum d on c.id=d.advertiser_conf_id 
+			where d.app_id =b.id
+			group by a.name,d.app_id ,d.agent_id ) h 
+			on c.app_id = h.app_id and c.agent_id = h.agent_id
+        where h.name is not null
+        ) xx
+        group by channel,wx_date
+        ) w on x.channel= w.channel
+        -----创建角色数
+  
+   left join
+       ( select channel ,
+    if(max(d_ct)=0,0,ifnull(max(e_ct),0)/max(d_ct)) as today_active_user_rate,
+    if(sum(a_ct)=0,0,ifnull(sum(b_ct),0)/sum(a_ct)) as second_stay_rate,
+    if(sum(a_ct)=0,0,ifnull(sum(c_ct),0)/sum(a_ct)) as third_stay_rate,
+    if(sum(a_ct)=0,0,ifnull(sum(seven_ct),0)/sum(a_ct)) as seven_stay_rate,
+    if(sum(a_ct)=0,0,ifnull(sum(fifteen_ct),0)/sum(a_ct)) as fifteen_stay_rate,
+    if(sum(a_ct)=0,0,ifnull(sum(thirty_ct),0)/sum(a_ct)) as thirty_stay_rate,
+    if(sum(a_ct)=0,0,ifnull(sum(fortyfive_ct),0)/sum(a_ct)) as fortyfive_stay_rate,
+    if(sum(a_ct)=0,0,ifnull(sum(sixty_ct),0)/sum(a_ct)) as sixty_stay_rate,
+    max(d_ct) game_user_sum from 
+        (select   h.name as channel ,
+        a.ct as a_ct,b.ct as b_ct,c.ct as c_ct,d.ct as d_ct,e.ct as e_ct,
+        seven.ct as seven_ct,fifteen.ct as fifteen_ct,thirty.ct as thirty_ct,
+        fortyfive.ct as fortyfive_ct,sixty.ct as sixty_ct
+        from 
+            (select a.name as name,d.app_id as app_id ,d.agent_id as agent_id
+            from ( select * from CostSourceData.advertiser_vx final) a
+			left join GameDataSum.h_game_sum b on a.book = b.name 
+			left join GameDataSum.mp_mp_conf_sum c on a.name= c.wx_name 
+			left join GameDataSum.mp_conf_agent_sum d on c.id=d.advertiser_conf_id 
+            where d.app_id =b.id 
+            group by a.name ,d.app_id ,d.agent_id) h 
+    left join 
+    (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        ( select * from GameDataSum.h_member_sum 
+                where create_time >={timestamp_ymd} 
+                    and create_time <={timestamp_tom}
+        ) a
+        left join 
+        (select DISTINCT mem_id mem_id  
+            from GameDataSum.h_log_mem_login_sum  
+            where date = '{ymd}' 
+            )  b on a.id=b.mem_id  
+        where b.mem_id !=''
+	group by a.app_id ,a.agent_id 
+	) a on a.app_id = h.app_id and a.agent_id = h.agent_id   
+    ----第一天
+    left join
+    (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        ( select * from GameDataSum.h_member_sum 
+                where create_time >={timestamp_ymd} 
+                    and create_time <={timestamp_tom}
+        ) a
+        left join 
+        (select DISTINCT mem_id  mem_id  
+            from GameDataSum.h_log_mem_login_sum  
+            where date = '{ymd_tom}' 
+            )  b on a.id=b.mem_id  
+        where b.mem_id !=''
+	group by a.app_id ,a.agent_id 
+	)  b on h.app_id =b.app_id and h.agent_id =b.agent_id
+     ----第二天
+     left join 
+     (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        ( select * from GameDataSum.h_member_sum 
+                where create_time >={timestamp_ymd} 
+                    and create_time <={timestamp_tom}
+        ) a
+        left join 
+        (select DISTINCT mem_id mem_id  
+            from GameDataSum.h_log_mem_login_sum  
+            where date = '{ymd_tom_after}' 
+            )  b on a.id=b.mem_id  
+        where b.mem_id !=''
+	group by a.app_id ,a.agent_id 
+	)  c on h.app_id =c.app_id and h.agent_id =c.agent_id
+    --第三天
+     left join 
+     (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        ( select * from GameDataSum.h_member_sum 
+                where create_time >={timestamp_ymd} 
+                    and create_time <={timestamp_tom}
+        ) a
+        left join 
+        (select DISTINCT mem_id mem_id  
+            from GameDataSum.h_log_mem_login_sum  
+            where date = '{ymd_seven_day}' 
+            )  b on a.id=b.mem_id  
+        where b.mem_id !=''
+	group by a.app_id ,a.agent_id 
+	)  seven on h.app_id = seven.app_id and h.agent_id = seven.agent_id
+
+    --第七天
+         left join 
+     (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        ( select * from GameDataSum.h_member_sum 
+                where create_time >={timestamp_ymd} 
+                    and create_time <={timestamp_tom}
+        ) a
+        left join 
+        (select DISTINCT mem_id  mem_id  
+            from GameDataSum.h_log_mem_login_sum  
+            where date = '{ymd_fifteen_day}' 
+            )  b on a.id=b.mem_id  
+        where b.mem_id !=''
+	group by a.app_id ,a.agent_id 
+	)  fifteen on h.app_id =fifteen.app_id and h.agent_id = fifteen.agent_id
+	
+    --第15天
+    left join 
+     (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        ( select * from GameDataSum.h_member_sum 
+                where create_time >={timestamp_ymd} 
+                    and create_time <={timestamp_tom}
+        ) a
+        left join 
+        (select DISTINCT mem_id mem_id  
+            from GameDataSum.h_log_mem_login_sum  
+            where date = '{ymd_thirty_day}' 
+            )  b on a.id=b.mem_id  
+        where b.mem_id !=''
+	group by a.app_id ,a.agent_id 
+	)  thirty on h.app_id =thirty.app_id and h.agent_id = thirty.agent_id
+    
+    --第30天
+     left join 
+     (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        ( select * from GameDataSum.h_member_sum 
+                where create_time >={timestamp_ymd} 
+                    and create_time <={timestamp_tom}
+        ) a
+        left join 
+        (select DISTINCT mem_id  mem_id  
+            from GameDataSum.h_log_mem_login_sum  
+            where date = '{ymd_fortyfive_day}' 
+            )  b on a.id=b.mem_id  
+        where b.mem_id !=''
+	group by a.app_id ,a.agent_id 
+	) fortyfive on h.app_id =fortyfive.app_id and h.agent_id = fortyfive.agent_id
+
+    --第45天
+    
+     left join 
+     (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        ( select * from GameDataSum.h_member_sum 
+                where create_time >={timestamp_ymd} 
+                    and create_time <={timestamp_tom}
+        ) a
+        left join 
+        (select DISTINCT mem_id  mem_id  
+            from GameDataSum.h_log_mem_login_sum  
+            where date = '{ymd_sixty_day}' 
+            )  b on a.id=b.mem_id  
+        where b.mem_id !=''
+	group by a.app_id ,a.agent_id 
+	) sixty on h.app_id =sixty.app_id and h.agent_id = sixty.agent_id
+
+    --第60天
+    left join 
+        (select app_id  ,count(*) as ct from GameDataSum.h_member_sum hm 
+        where  create_time <={timestamp_tom}
+        group by app_id  ) d on h.app_id =d.app_id
+    --游戏用户数量
+    left join 
+        (select count(distinct(mem_id)) as ct,app_id  from GameDataSum.h_log_mem_login_sum 
+       where date = '{ymd}' 
+       group by app_id  ) e on  h.app_id =e.app_id
+    --今日活跃用户数量
+    )
+    as keep_data
+    group by channel)   v on x.channel= v.channel
+  
+  
+  '''
+
+    print(ck_sql)
+    ck.execute(ck_sql)
+
+
+def dw_daily_channel_cost_ck(ymd):
+    def table_name(datatime_tmp, datatime_realtime):
+        datatime_use = min(datatime_tmp, datatime_realtime)
+        str_year = datatime_use.tm_year
+        str_mon = datatime_use.tm_mon
+        str_mon = str_mon if str_mon > 9 else '0' + str(str_mon)
+        res = 'h_log_mem_login_{}{}'.format(str_year, str_mon)
+        return res
+
+    logging.info("run> dw_daily_channel_cost")
+    datatime_ymd = datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))).timetuple()
+    datatime_ymd_tom = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=1)).timetuple()
+    datatime_ymd_tom_after = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=2)).timetuple()
+    datatime_ymd_seven_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=7)).timetuple()
+    datatime_ymd_fifteen_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=15)).timetuple()
+    datatime_ymd_thirty_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=30)).timetuple()
+    datatime_ymd_fortyfive_day = (
+            datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=45)).timetuple()
+    datatime_ymd_sixty_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=60)).timetuple()
+    datatime_realtime = datetime.now().timetuple()
+
+    # datatime_str
+    ymd_tom = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=1)).strftime('%Y-%m-%d')
+    ymd_tom_after = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=2)).strftime('%Y-%m-%d')
+    ymd_seven_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=7)).strftime('%Y-%m-%d')
+    ymd_fifteen_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=15)).strftime('%Y-%m-%d')
+    ymd_thirty_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=30)).strftime('%Y-%m-%d')
+    ymd_fortyfive_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=45)).strftime('%Y-%m-%d')
+    ymd_sixty_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=60)).strftime('%Y-%m-%d')
+
+    # timestamp
+    timestamp_ymd = time.mktime(datatime_ymd)
+    timestamp_tom = time.mktime(datatime_ymd_tom)
+
+    # table_name
+    table_name_login_today = table_name(datatime_ymd, datatime_realtime)
+    table_name_login_tom = table_name(datatime_ymd_tom, datatime_realtime)
+    table_name_login_tom_after = table_name(datatime_ymd_tom_after, datatime_realtime)
+    table_name_login_seven_day = table_name(datatime_ymd_seven_day, datatime_realtime)
+    table_name_login_fifteen_day = table_name(datatime_ymd_fifteen_day, datatime_realtime)
+    table_name_login_thirty_day = table_name(datatime_ymd_thirty_day, datatime_realtime)
+    table_name_login_fortyfive_day = table_name(datatime_ymd_fortyfive_day, datatime_realtime)
+    table_name_login_sixty_day = table_name(datatime_ymd_sixty_day, datatime_realtime)
+
+    database_names = {'GameDataTwoDbMpPart', 'GameDataOneDbMpPart'}
+
+    for database_name in database_names:
+        ck_sql = f'''
+    insert into game_data.dw_daily_channel_cost 
+        (dt,channel,pitcher,stage,platform,book,view_count,click_count,follow_user,cost,
+        web_view_count,platform_view_count,web_order_count,type,require_roi,require_mult,
+        reg_num,create_user_num,today_active_user_rate,second_stay_rate,third_stay_rate,
+        seven_stay_rate,fifteen_stay_rate,thirty_stay_rate,fortyfive_stay_rate,
+        sixty_stay_rate,game_user_sum)
+    select x.dt,x.channel,pitcher,stage,x.platform,x.book,
+               ifnull(view_count,0),ifnull(click_count,0),
+               ifnull(follow_user,0),ifnull(cost,0)/100 as cost,
+               ifnull(web_view_count,0) web_view_count,
+               ifnull(platform_view_count,0) platform_view_count,
+               ifnull(web_order_count,0) web_order_count,
+               if(stage ='趣程15期' or stage ='趣程26期' or stage ='趣程30期','GDT','MP') type
+                ,0 require_roi,0 require_mult,
+                ifnull(y.reg_num,0),ifnull(w.create_user_num,0),
+                v.today_active_user_rate,
+                v.second_stay_rate,
+                v.third_stay_rate,
+                v.seven_stay_rate, 
+                v.fifteen_stay_rate,
+                v.thirty_stay_rate,
+                v.fortyfive_stay_rate,
+                v.sixty_stay_rate,
+                v.game_user_sum
+        from
+        ( select dt, channel,stage,pitcher,platform,book  from CostSourceData.channel_info_daily cid 
+            where dt='{ymd}' and channel !=''
+            and channel in 
+            (select distinct(channel) from CostSourceData.channel_by_account_daily cbad 
+            where dt='{ymd}'
+            and (type ='GDT' or type='MP')
+            ) 
+        ) x  -- 只允许渠道MP、GDT
+        left join
+        
+        ( select channel,
+		sum(cost) as cost,
+		sum(view_count) as view_count,
+		sum(valid_click_count) as click_count,
+          sum(from_follow_uv)  as follow_user,
+         sum(web_view_count) as web_view_count,
+         sum(platform_view_count) as platform_view_count,
+         sum(web_order_count) as web_order_count
+         from
+            (select account_id,cost,view_count,valid_click_count,
+            round(valid_click_count*official_account_follow_rate,0) as from_follow_uv,
+            0 as  web_view_count,
+            0 as platform_view_count,
+            0 as web_order_count
+             from CostSourceData.daily_vx
+             where date=toDateTime('{ymd} 00:00:00')
+            union all
+            select account_id,cost,view_count,valid_click_count,from_follow_uv,
+            ifnull(web_commodity_page_view_count,0) as  web_view_count, 
+            ifnull(platform_page_view_count,0) as platform_view_count,
+            ifnull(web_order_count,0) as web_order_count
+             from CostSourceData.daily_qq
+             where date=toDateTime('{ymd} 00:00:00')
+             ) a
+        left join
+        (select toString(account_id) account_id,channel 
+        from CostSourceData.channel_by_account_daily 
+        			where dt=toDate('{ymd}')) b 
+         on a.account_id=b.account_id  group by channel) 
+        z on x.channel=z.channel   -- 只允许渠道cost  消耗
+  
+  
+   left join     
+        (
+        select channel,wx_date,count(*) reg_num
+        from  (
+         select h.name as channel ,toDate(create_time) wx_date
+        from 
+        {database_name}.h_member origin left join
+        (select a.name name,d.app_id app_id ,d.agent_id agent_id from CostSourceData.advertiser_vx a
+			left join {database_name}.h_game b on a.book = b.name 
+			left join {database_name}.mp_mp_conf c on a.name= c.wx_name 
+			left join {database_name}.mp_conf_agent d on c.id=d.advertiser_conf_id 
+			where d.app_id =b.id
+			group by a.name,d.app_id ,d.agent_id ) h 
+			on origin.app_id = h.app_id and origin.agent_id = h.agent_id
+            where h.name is not null      
+            and origin.create_time > {timestamp_ymd} and origin.create_time < {timestamp_tom}  
+        ) tmp
+        group by channel, wx_date
+        ) y on x.channel= y.channel  
+        --- 注册用户数
+  
+  
+   left join     
+        (
+        select  channel, wx_date,
+        count(*) as create_user_num from 
+        (select h.name as channel,DATE(FROM_UNIXTIME(c.create_time)) as wx_date
+        from {database_name}.h_mg_role a
+        left join {database_name}.h_mem_game b on a.mg_mem_id = b.id
+        left join {database_name}.h_member c on b.mem_id = c.id
+        left join
+        (select a.name name ,d.app_id app_id ,d.agent_id agent_id from CostSourceData.advertiser_vx a
+			left join {database_name}.h_game b on a.book = b.name 
+			left join {database_name}.mp_mp_conf c on a.name= c.wx_name 
+			left join {database_name}.mp_conf_agent d on c.id=d.advertiser_conf_id 
+			where d.app_id =b.id
+			group by a.name,d.app_id ,d.agent_id ) h 
+			on c.app_id = h.app_id and c.agent_id = h.agent_id
+        where h.name is not null
+                and c.create_time >= {timestamp_ymd} and c.create_time <= {timestamp_tom}
+        ) xx
+        group by channel,wx_date
+        ) w on x.channel= w.channel
+        -----创建角色数
+  
+   left join
+       ( select channel ,
+    if(max(d_ct)=0,0,ifnull(max(e_ct),0)/max(d_ct)) as today_active_user_rate,
+    if(sum(a_ct)=0,0,ifnull(sum(b_ct),0)/sum(a_ct)) as second_stay_rate,
+    if(sum(a_ct)=0,0,ifnull(sum(c_ct),0)/sum(a_ct)) as third_stay_rate,
+    if(sum(a_ct)=0,0,ifnull(sum(seven_ct),0)/sum(a_ct)) as seven_stay_rate,
+    if(sum(a_ct)=0,0,ifnull(sum(fifteen_ct),0)/sum(a_ct)) as fifteen_stay_rate,
+    if(sum(a_ct)=0,0,ifnull(sum(thirty_ct),0)/sum(a_ct)) as thirty_stay_rate,
+    if(sum(a_ct)=0,0,ifnull(sum(fortyfive_ct),0)/sum(a_ct)) as fortyfive_stay_rate,
+    if(sum(a_ct)=0,0,ifnull(sum(sixty_ct),0)/sum(a_ct)) as sixty_stay_rate,
+    max(d_ct) game_user_sum from 
+        (select   h.name as channel ,
+        a.ct as a_ct,b.ct as b_ct,c.ct as c_ct,d.ct as d_ct,e.ct as e_ct,
+        seven.ct as seven_ct,fifteen.ct as fifteen_ct,thirty.ct as thirty_ct,
+        fortyfive.ct as fortyfive_ct,sixty.ct as sixty_ct
+        from 
+            (select a.name as name,d.app_id as app_id ,d.agent_id as agent_id
+			from CostSourceData.advertiser_vx a
+			left join {database_name}.h_game b on a.book = b.name 
+			left join {database_name}.mp_mp_conf c on a.name= c.wx_name 
+			left join {database_name}.mp_conf_agent d on c.id=d.advertiser_conf_id 
+            where d.app_id =b.id
+            group by a.name ,d.app_id ,d.agent_id) h 
+    left join 
+    (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        {database_name}.h_member  a
+        left join 
+        (select toUInt64(mem_id) mem_id from 
+            (select Distinct(mem_id) mem_id,date
+            from {database_name}.{table_name_login_today}  
+            where date = '{ymd}' 
+            ) )  b on a.id=b.mem_id  
+        where a.create_time >={timestamp_ymd} and a.create_time <={timestamp_tom}
+        and b.mem_id !=0
+	group by a.app_id ,a.agent_id 
+	) a on a.app_id = h.app_id and a.agent_id = h.agent_id   
+    ----第一天
+    left join 
+    (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        {database_name}.h_member  a
+        left join 
+        (select toUInt64(mem_id) mem_id from 
+            (select Distinct(mem_id) mem_id,date
+            from {database_name}.{table_name_login_tom}  
+            where date = '{ymd_tom}' 
+            ) )  b on a.id=b.mem_id  
+        where a.create_time >={timestamp_ymd} and a.create_time <={timestamp_tom}
+        and b.mem_id !=0
+	group by a.app_id ,a.agent_id 
+	) b on h.app_id =b.app_id and h.agent_id =b.agent_id
+     ----第二天
+    left join 
+    (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        {database_name}.h_member  a
+        left join 
+        (select toUInt64(mem_id) mem_id from 
+            (select Distinct(mem_id) mem_id,date
+            from {database_name}.{table_name_login_tom_after}  
+            where date = '{ymd_tom_after}' 
+            ) )  b on a.id=b.mem_id  
+        where a.create_time >={timestamp_ymd} and a.create_time <={timestamp_tom}
+        and b.mem_id !=0
+	group by a.app_id ,a.agent_id 
+	) c on h.app_id =c.app_id and h.agent_id = c.agent_id
+    --第三天
+    left join 
+    (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        {database_name}.h_member  a
+        left join 
+        (select toUInt64(mem_id) mem_id from 
+            (select Distinct(mem_id) mem_id,date
+            from {database_name}.{table_name_login_seven_day}  
+            where date = '{ymd_seven_day}' 
+            ) )  b on a.id=b.mem_id  
+        where a.create_time >={timestamp_ymd} and a.create_time <={timestamp_tom}
+        and b.mem_id !=0
+	group by a.app_id ,a.agent_id 
+	)  seven on h.app_id = seven.app_id and h.agent_id = seven.agent_id
+    --第七天
+    left join 
+     (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        {database_name}.h_member  a
+        left join 
+        (select toUInt64(mem_id) mem_id from 
+            (select Distinct(mem_id) mem_id,date
+            from {database_name}.{table_name_login_fifteen_day}  
+            where date = '{ymd_fifteen_day}' 
+            ) )  b on a.id=b.mem_id  
+        where a.create_time >={timestamp_ymd} and a.create_time <={timestamp_tom}
+        and b.mem_id !=0
+	group by a.app_id ,a.agent_id 
+	)  fifteen on h.app_id =fifteen.app_id and h.agent_id = fifteen.agent_id
+    --第15天
+    left join
+    (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        {database_name}.h_member  a
+        left join 
+        (select toUInt64(mem_id) mem_id from 
+            (select Distinct(mem_id) mem_id,date
+            from {database_name}.{table_name_login_thirty_day}  
+            where date = '{ymd_thirty_day}' 
+            ) )  b on a.id=b.mem_id  
+        where a.create_time >={timestamp_ymd} and a.create_time <={timestamp_tom}
+        and b.mem_id !=0
+	group by a.app_id ,a.agent_id 
+	) thirty on h.app_id =thirty.app_id and h.agent_id = thirty.agent_id
+    --第30天
+    left join
+    (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        {database_name}.h_member  a
+        left join 
+        (select toUInt64(mem_id) mem_id from 
+            (select Distinct(mem_id) mem_id,date
+            from {database_name}.{table_name_login_fortyfive_day}  
+            where date = '{ymd_fortyfive_day}' 
+            ) )  b on a.id=b.mem_id  
+        where a.create_time >={timestamp_ymd} and a.create_time <={timestamp_tom}
+        and b.mem_id !=0
+	group by a.app_id ,a.agent_id 
+	) fortyfive on h.app_id =fortyfive.app_id and h.agent_id = fortyfive.agent_id
+    --第45天
+     left join
+    (select '{ymd}',a.app_id,a.agent_id,count(*) ct from 
+        {database_name}.h_member  a
+        left join 
+        (select toUInt64(mem_id) mem_id from 
+            (select Distinct(mem_id) mem_id,date
+            from {database_name}.{table_name_login_sixty_day}  
+            where date = '{ymd_sixty_day}' 
+            ) )  b on a.id=b.mem_id  
+        where a.create_time >={timestamp_ymd} and a.create_time <={timestamp_tom}
+       and b.mem_id !=0
+	group by a.app_id ,a.agent_id 
+	)  sixty on h.app_id =sixty.app_id and h.agent_id = sixty.agent_id
+    --第60天
+    left join 
+        (select app_id  ,count(*) as ct from {database_name}.h_member hm 
+        where  create_time <={timestamp_tom}
+        group by app_id  ) d on h.app_id =d.app_id
+    --游戏用户数量
+    left join 
+        (select count(distinct(mem_id)) as ct,app_id  from {database_name}.{table_name_login_today} 
+       where date = '{ymd}' 
+       group by app_id  ) e on  h.app_id =e.app_id
+    --今日活跃用户数量
+    )
+    as keep_data
+    group by channel)   v on x.channel= v.channel
+  
+  
+  '''
+
+        print(ck_sql)
+        ck.execute(ck_sql)
+
+
 def dw_daily_channel_cost(ymd):
     def table_name(datatime_tmp, datatime_realtime):
         datatime_use = min(datatime_tmp, datatime_realtime)
@@ -42,8 +687,8 @@ def dw_daily_channel_cost(ymd):
     datatime_ymd_thirty_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
         days=30)).timetuple()
     datatime_ymd_fortyfive_day = (
-                datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
-            days=45)).timetuple()
+            datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
+        days=45)).timetuple()
     datatime_ymd_sixty_day = (datetime.strptime(ymd, '%Y-%m-%d').astimezone(timezone(timedelta(hours=8))) + timedelta(
         days=60)).timetuple()
     datatime_realtime = datetime.now().timetuple()
@@ -274,10 +919,9 @@ group by channel)
         
         
         """
-    print(sql)
+    # print(sql)
     data = db.quchen_text.get_data_list(sql)
 
-
     data1 = []
     col = "dt,channel,pitcher,stage,platform,book,view_count,click_count,follow_user,cost,web_view_count,platform_view_count,web_order_count,type,require_roi,require_mult,reg_num,create_user_num,today_active_user_rate,second_stay_rate,third_stay_rate,seven_stay_rate,fifteen_stay_rate,thirty_stay_rate,fortyfive_stay_rate,sixty_stay_rate,game_user_sum"
     for i in data:
@@ -425,11 +1069,13 @@ if __name__ == '__main__':
     # dw_daily_channel_cost('2021-09-19')
     # exit()
     # channel_by_account_daily('2021-02-05')
-    for i in dt.getDateLists('2021-09-08', '2021-09-30'):
-        print(i)
-        channel_by_account_daily(i)
-        channel_info_daily(i)
-
-        dw_daily_channel_cost(i)
-        # ods_order('2021-05-06')
-        platform_data_sum(i)
+    for i in dt.getDateLists('2021-09-01', '2022-01-02'):
+        # print(i)
+        # channel_by_account_daily(i)
+        # channel_info_daily(i)
+        #
+        # dw_daily_channel_cost(i)
+        # # ods_order('2021-05-06')
+        # platform_data_sum(i)
+        # dw_daily_channel_cost_ck(i)
+        dw_daily_channel_cost_ck_gamedatasum(i)

app/etl/dw/dw_channel_daily.py

@@ -3,6 +3,8 @@
 
 @auth ck
 """
+import time
+
 from model.DateUtils import DateUtils
 from model.DataBaseUtils import MysqlUtils, CkUtils
 import logging
@@ -218,7 +220,8 @@ def dw_channel_amount_daily_reverse():
                 sum(if(addDays(date,4)=today(),amount,0)) ba5,
                 sum(if(addDays(date,6)=today(),amount,0)) ba6,
                 sum(if(addDays(date,7)=today(),amount,0)) ba7
-     from game_data.order where status=2 and reg_time>'2019-03-18 00:00:00' and date>=subtractDays(today(),7) group by toDate(formatDateTime(reg_time,'%Y-%m-%d')),channel"""
+     from game_data.order where status=2 and reg_time>'2019-03-18 00:00:00'
+      and date>=subtractDays(today(),7) group by toDate(formatDateTime(reg_time,'%Y-%m-%d')),channel"""
     data = ck.execute(sql)
     isql = "replace into dw_channel_amount_daily_reverse values (%s,%s,%s,%s,%s,%s,%s,%s,%s)"
     db.dm.execute("truncate table dw_channel_amount_daily_reverse")

config/db_config.yaml

@@ -30,6 +30,7 @@ zx_platform:
 
 dm:
   host: qc-game-cluster.rwlb.rds.aliyuncs.com
+  port: 3306
   user: qc
   passwd: Qc_1234567
   db: dm
@@ -49,9 +50,9 @@ zx_test:
 
 
 clickhouse:
-  host: cc-bp11803zbt0oq045io.ads.rds.aliyuncs.com
+  host: cc-bp1ah87xjy8yb884s.public.clickhouse.ads.aliyuncs.com
   user: qc
-  passwd: Qc_123456
+  passwd: Qc_1234567
   port: 3306
 
 #clickhouse:

model/DataBaseUtils.py

@@ -91,6 +91,7 @@ class MysqlUtils:
         self._dm = MysqlOperation(host=conf['host'],
                                   user=conf['user'],
                                   passwd=conf['passwd'],
+                                  port=conf['port'],
                                   db=conf['db'])
         return self._dm
 
@@ -134,7 +135,7 @@ class CkUtils:
                              user=conf['user'],
                              password=conf['passwd'],
                              port=conf['port'],
-                             send_receive_timeout=5)
+                             send_receive_timeout=600)
 
     def execute(self, sql):