MOD:游戏号数据拉取

app/api_data/platform_order/order_data_change.py

@@ -48,7 +48,7 @@ def insert_order_data_hourly():
 
 
 def insert_order_data_daily():
-    for i in du.getDateLists(du.get_n_days(-40), du.get_n_days(0)):
+    for i in du.getDateLists(du.get_n_days(-10), du.get_n_days(0)):
         print(i)
         insert_order_data(i)
 

app/api_data/tx_ad_cost/get_cost_older.py

@@ -8,6 +8,7 @@ from concurrent.futures import ThreadPoolExecutor
 from model.DateUtils import DateUtils
 from model.DataBaseUtils import MysqlUtils
 from model.DingTalkUtils import DingTalkUtils
+from app.api_data.tx_ad_cost.get_cost_game import get_data
 from six import string_types
 from six.moves.urllib.parse import urlencode, urlunparse
 
@@ -75,15 +76,14 @@ def get_q_data(y, li, st, et):
                 d['account_id'] = y[0]
                 logging.info('qq: ' + str(d['account_id']) + str(d["cost"]))
                 x = d
-                res_data=[x['date'],x['view_count'],x['valid_click_count'],x['ctr'],x['cpc'],x['cost'],
-                          x['web_order_count'],x['web_order_rate'],
-                          x['web_order_cost'],x['follow_count'],x['order_amount'],x['order_roi'],
-                          x['platform_page_view_count'],x['web_commodity_page_view_count'],
-                          x['from_follow_uv'],x['account_id']]
+                res_data = [x['date'], x['view_count'], x['valid_click_count'], x['ctr'], x['cpc'], x['cost'],
+                            x['web_order_count'], x['web_order_rate'],
+                            x['web_order_cost'], x['follow_count'], x['order_amount'], x['order_roi'],
+                            x['platform_page_view_count'], x['web_commodity_page_view_count'],
+                            x['from_follow_uv'], x['account_id']]
                 li.append(tuple(res_data))
     except Exception as e:
-        logging.error('qq account:{}   error :{}'.format(str(y),str(e)))
-
+        logging.error('qq account:{}   error :{}'.format(str(y), str(e)))
 
 
 def get_v_data(y, li, st, et):
@@ -96,24 +96,31 @@ def get_v_data(y, li, st, et):
         if 'data' in c.keys() and len(c["data"]["list"]) > 0:
             for d in c['data']['list']:
                 d['account_id'] = y[0]
-                logging.info('vx:' + str(d['account_id'])+ ' ' + str(d["cost"]))
+                logging.info('vx:' + str(d['account_id']) + ' ' + str(d["cost"]))
                 x = d
-                res_data=[x['date'],x['cost'],x['view_count'],x['valid_click_count'],x['ctr'],
-                          x['official_account_follow_rate'],
-                          x['order_amount'],x['order_roi'],x['order_count'],x['order_rate'],
-                          x['order_unit_price'],x['web_order_cost'],x['first_day_order_amount'],
-                          x['first_day_order_count'],x['account_id']]
+                res_data = [x['date'], x['cost'], x['view_count'], x['valid_click_count'], x['ctr'],
+                            x['official_account_follow_rate'],
+                            x['order_amount'], x['order_roi'], x['order_count'], x['order_rate'],
+                            x['order_unit_price'], x['web_order_cost'], x['first_day_order_amount'],
+                            x['first_day_order_count'], x['account_id']]
                 li.append(tuple(res_data))
     except Exception as e:
-        logging.error('vx account:{}   error :{}'.format(str(y),str(e)))
+        logging.error('vx account:{}   error :{}'.format(str(y), str(e)))
 
 
+def get_vx_game_list():
+    sql = '''select account_id,wechat_account_id,access_token,refresh_token,name,
+          ifnull(stage,''),ifnull(pitcher,''),ifnull(platform,''),ifnull(book,'') from advertiser_vx
+          where access_token is not null and is_game=1
+          '''
+    a = db.quchen_text.getData(sql)
+    return a
 
 
 def get_vx_list():
     sql = '''select account_id,wechat_account_id,access_token,refresh_token,name,
           ifnull(stage,''),ifnull(pitcher,''),ifnull(platform,''),ifnull(book,'') from advertiser_vx
-          where access_token is not null
+          where access_token is not null and is_game=0
           '''
     a = db.quchen_text.getData(sql)
     return a
@@ -142,8 +149,6 @@ def mysql_insert_daily_qq(data):
     db.quchen_text.executeMany(a, data)
 
 
-
-
 def get_daily_vx(st, et):
     token_list_v = get_vx_list()
     logging.info("获取vx账号：" + str(token_list_v.__len__()))
@@ -158,6 +163,20 @@ def get_daily_vx(st, et):
     mysql_insert_daily_vx(li)
 
 
+def get_daily_vx_game(st, et):
+    token_list_v = get_vx_game_list()
+    logging.info("获取vx_game账号：" + str(token_list_v.__len__()))
+    time1 = time.time()
+    executor = ThreadPoolExecutor(max_workers=10)
+    li = []
+    for y in token_list_v:
+        executor.submit(get_data, y, li, st, et)
+    executor.shutdown()
+    logging.info('get_daily_vx:' + str(len(li)) + 'cost:' + str(int(time.time() - time1)))
+
+    mysql_insert_daily_vx(li)
+
+
 def get_daily_qq(st, et):
     token_list_q = get_qq_list()
     logging.info("获取qq账号：" + str(token_list_q.__len__()))
@@ -175,6 +194,10 @@ def run(st, et):
     logging.info('微信消耗数据拉取,开始')
     get_daily_vx(st, et)
     logging.info('微信消耗数据拉取,结束')
+    logging.info('微信游戏消耗数据拉取,结束')
+    get_daily_vx_game(st, et)
+    logging.info('微信游戏消耗数据拉取,结束')
+
     # logging.info('qq消耗数据拉取,开始')
     # get_daily_qq(st, et)
     # logging.info('qq消耗数据拉取,结束')
@@ -196,6 +219,6 @@ def old_cost_daily():
 if __name__ == '__main__':
     # run()
     # old_cost_daily()
-    st = du.get_n_days(-365)
+    st = du.get_n_days(-10)
     et = du.get_n_days(0)
     run(st, et)

app/etl/data_stat_task.py

@@ -218,12 +218,11 @@ def channel_info_daily(ymd):
             select  name from book_change group by name) a
              
              left join
-              (
-                select name,ifnull(stage,'') stage,ifnull(pitcher,'') pitcher,
+              ( select name,ifnull(stage,'') stage,ifnull(pitcher,'') pitcher,
                 ifnull(platform,'') platform,ifnull(book,'') book 
                 from advertiser_vx 
                 where name is not null  
-                and start_date < '{ymd}'
+                and start_date <= '{ymd}'
                 group by name,stage,pitcher,platform,book
                 ) b on a.name=b.name
                 """
@@ -304,4 +303,4 @@ if __name__ == '__main__':
 
         dw_daily_channel_cost(i)
         # ods_order('2021-05-06')
-        platform_data_sum(i)
+        platform_data_sum(i)

app/etl/dw/dw_channel_daily.py

@@ -78,6 +78,7 @@ left outer join
     # db.dm.execute("truncate table dw_channel")
     db.dm.executeMany(isql, data)
 
+
 def dw_channel_user_daily():
     sql = """
 select toDate(formatDateTime(reg_time,'%Y-%m-%d')) as dt,