MOD:change for tencent data

app/api_data/tx_ad_cost/get_cost_older.py

@@ -65,30 +65,47 @@ def daily_reports_get(access_token, account_id, st, et, level, fields, err_num=0
 
 
 def get_q_data(y, li, st, et):
-    c = daily_reports_get(y[2], y[0], st, et, "REPORT_LEVEL_ADVERTISER", (
-        'date', 'view_count', 'valid_click_count', 'ctr', 'cpc', 'cost', 'web_order_count', 'web_order_rate',
-        'web_order_cost', 'follow_count', 'order_amount', 'order_roi', 'platform_page_view_count',
-        'web_commodity_page_view_count', 'from_follow_uv'))
-    if 'data' in c.keys() and len(c["data"]["list"]) > 0:
-        for d in c['data']['list']:
-            d['account_id'] = y[0]
-            logging.info('qq: ' + str(d['account_id']) + str(d["cost"]))
-            x = tuple(d.values())
-            li.append(x)
+    try:
+        c = daily_reports_get(y[2], y[0], st, et, "REPORT_LEVEL_ADVERTISER", (
+            'date', 'view_count', 'valid_click_count', 'ctr', 'cpc', 'cost', 'web_order_count', 'web_order_rate',
+            'web_order_cost', 'follow_count', 'order_amount', 'order_roi', 'platform_page_view_count',
+            'web_commodity_page_view_count', 'from_follow_uv'))
+        if 'data' in c.keys() and len(c["data"]["list"]) > 0:
+            for d in c['data']['list']:
+                d['account_id'] = y[0]
+                logging.info('qq: ' + str(d['account_id']) + str(d["cost"]))
+                x = d
+                res_data=[x['date'],x['view_count'],x['valid_click_count'],x['ctr'],x['cpc'],x['cost'],
+                          x['web_order_count'],x['web_order_rate'],
+                          x['web_order_cost'],x['follow_count'],x['order_amount'],x['order_roi'],
+                          x['platform_page_view_count'],x['web_commodity_page_view_count'],
+                          x['from_follow_uv'],x['account_id']]
+                li.append(tuple(res_data))
+    except Exception as e:
+        logging.error('qq account:{}   error :{}'.format(str(y),str(e)))
 
 
-def get_v_data(y, li, st, et):
-    c = daily_reports_get(y[2], y[0], st, et, "REPORT_LEVEL_ADVERTISER_WECHAT", (
-        'date', 'cost', 'view_count', 'valid_click_count', 'ctr', 'official_account_follow_rate', 'order_amount',
-        'order_roi', 'order_count', 'order_rate', 'order_unit_price', 'web_order_cost', 'first_day_order_amount',
-        'first_day_order_count'))
 
-    if 'data' in c.keys() and len(c["data"]["list"]) > 0:
-        for d in c['data']['list']:
-            d['account_id'] = y[0]
-            logging.info('vx:' + str(d['account_id']) + str(d["cost"]))
-            x = tuple(d.values())
-            li.append(x)
+def get_v_data(y, li, st, et):
+    try:
+        c = daily_reports_get(y[2], y[0], st, et, "REPORT_LEVEL_ADVERTISER_WECHAT", (
+            'date', 'cost', 'view_count', 'valid_click_count', 'ctr', 'official_account_follow_rate', 'order_amount',
+            'order_roi', 'order_count', 'order_rate', 'order_unit_price', 'web_order_cost', 'first_day_order_amount',
+            'first_day_order_count'))
+
+        if 'data' in c.keys() and len(c["data"]["list"]) > 0:
+            for d in c['data']['list']:
+                d['account_id'] = y[0]
+                logging.info('vx:' + str(d['account_id'])+ ' ' + str(d["cost"]))
+                x = d
+                res_data=[x['date'],x['cost'],x['view_count'],x['valid_click_count'],x['ctr'],
+                          x['official_account_follow_rate'],
+                          x['order_amount'],x['order_roi'],x['order_count'],x['order_rate'],
+                          x['order_unit_price'],x['web_order_cost'],x['first_day_order_amount'],
+                          x['first_day_order_count'],x['account_id']]
+                li.append(tuple(res_data))
+    except Exception as e:
+        logging.error('vx account:{}   error :{}'.format(str(y),str(e)))
 
 
 def get_tt_data(account_info, li, st, et):
@@ -162,10 +179,12 @@ def get_qq_list():
 
 
 def mysql_insert_daily_vx(data):
+    logging.info('start save daily_vx info')
     b = """replace into daily_vx (date,cost,view_count,valid_click_count,ctr,official_account_follow_rate,order_amount,
 	order_roi,order_count,order_rate,order_unit_price,web_order_cost,first_day_order_amount,first_day_order_count,account_id)
 	 values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"""
     db.quchen_text.executeMany(b, data)
+    logging.info('start save daily_vx info')
 
 
 def mysql_insert_daily_qq(data):
@@ -225,7 +244,6 @@ def get_daily_tt(st, et):
             "refresh_token": refresh_token,
         }
         rsp = requests.post(refresh_token_url, json=data)
-        # print(rsp.text)
         rsp_data = rsp.json()
         new_refresh_token = rsp_data['data']['refresh_token']
         new_access_token = rsp_data['data']['access_token']

app/etl/data_stat_task.py

@@ -10,7 +10,7 @@ dt = DateUtils()
 def dw_daily_bytedance_cost(ymd):
     logging.info(f'dw_daily_bytedance_cost 数据填充开始')
 
-    sql='''
+    sql = '''
     select x.dt,x.channel,pitcher,stage,x.platform,x.book,
                ifnull(view_count,0),ifnull(click_count,0),
                ifnull(follow_user,0),ifnull(cost,0)/100 as cost,
@@ -64,10 +64,11 @@ def dw_daily_bytedance_cost(ymd):
 
     logging.info(f'dw_daily_bytedance_cost 数据填充结束')
 
+
 def platform_data_sum(ymd):
     logging.info('dw_daily_platform_cost开始数据更新')
     ck.execute("alter table dw_daily_platform_cost drop  partition '{}' ".format(ymd))
-    sql=f'''
+    sql = f'''
         insert into dw_daily_platform_cost
         select * from 
         (select * from dw_daily_bytedance_cost a where dt='{ymd}'