MOD:添加消耗广告数据埋点

app/api_data/tx_ad_cost/cost_util.py

@@ -7,6 +7,7 @@ from concurrent.futures import ThreadPoolExecutor
 from model.DataBaseUtils import MysqlUtils
 from model.ComUtils import *
 from model.DateUtils import DateUtils
+
 du = DateUtils()
 db = MysqlUtils()
 max_workers = 10
@@ -14,10 +15,10 @@ count = []
 t = du.get_n_days(-10)
 
 
-def get_campaign(account_id, access_token, flag,campaign_ids,dt):
+def get_campaign(account_id, access_token, flag, campaign_ids, dt):
     path = 'campaigns/get'
     fields = ('campaign_id', 'campaign_name', 'configured_status', 'campaign_type', 'promoted_object_type',
-              'daily_budget','budget_reach_date','created_time','last_modified_time','speed_mode','is_deleted')
+              'daily_budget', 'budget_reach_date', 'created_time', 'last_modified_time', 'speed_mode', 'is_deleted')
     url = 'https://api.e.qq.com/v1.3/' + path
     li = []
     page = 1
@@ -62,24 +63,23 @@ def get_campaign(account_id, access_token, flag,campaign_ids,dt):
         if r.get("data"):
             for i in r['data']['list']:
                 li.append((str(i['campaign_id']), i['campaign_name'], i['configured_status'], i['campaign_type'],
-                           i['promoted_object_type'],i['daily_budget'],i.get('budget_reach_date'),
+                           i['promoted_object_type'], i['daily_budget'], i.get('budget_reach_date'),
                            DateUtils.stamp_to_str(i['created_time']),
-                           DateUtils.stamp_to_str(i['last_modified_time']),i.get('speed_mode'),i.get('is_deleted'),
+                           DateUtils.stamp_to_str(i['last_modified_time']), i.get('speed_mode'), i.get('is_deleted'),
                            account_id, flag, dt))
     # logging.info(li)
     """mp 没有 speed_mode，is_deleted，budget_reach_date"""
     if li.__len__() > 0:
-        logging.info(f"{account_id}有计划:"+ str(li.__len__()))
+        logging.info(f"{account_id}有计划:" + str(li.__len__()))
         sql = "replace into campaign_info values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
         db.quchen_text.executeMany(sql, li)
     db.close()
 
 
-
-def get_adcreatives(account_id,access_token,flag,adc_ids,dt):  # 获取创意
-    #接口https://developers.e.qq.com/docs/api/adsmanagement/adcreatives/adcreatives_get?version=1.3
+def get_adcreatives(account_id, access_token, flag, adc_ids, dt):  # 获取创意
+    # 接口https://developers.e.qq.com/docs/api/adsmanagement/adcreatives/adcreatives_get?version=1.3
     url = 'https://api.e.qq.com/v1.1/adcreatives/get'
-    li =[]
+    li = []
     page = 1
     logging.info(f"{account_id}开始获取创意")
     while True:
@@ -88,15 +88,15 @@ def get_adcreatives(account_id,access_token,flag,adc_ids,dt):  # 获取创意
             'access_token': access_token,
             'timestamp': int(time.time()),
             'nonce': str(time.time()) + str(random.randint(0, 999999)),
-            'fields': ('campaign_id', 'adcreative_id', 'adcreative_name', 'adcreative_elements', 'promoted_object_type', 'page_type',
-            'page_spec', 'link_page_spec', 'universal_link_url', 'promoted_object_id','site_set'),
+            'fields': ('campaign_id', 'adcreative_id', 'adcreative_name', 'adcreative_elements', 'promoted_object_type',
+                       'page_type',
+                       'page_spec', 'link_page_spec', 'universal_link_url', 'promoted_object_id', 'site_set'),
             "filtering": [{
                 "field": "adcreative_id",
                 "operator": "IN",
                 "values": adc_ids.split(',')
 
-
-                    }],
+            }],
             "account_id": account_id,
             "page": page,
             "page_size": 100,
@@ -107,9 +107,8 @@ def get_adcreatives(account_id,access_token,flag,adc_ids,dt):  # 获取创意
             if type(parameters[k]) is not str:
                 parameters[k] = json.dumps(parameters[k])
 
-
         while True:
-            h = requests.get(url, params=parameters,timeout=10)
+            h = requests.get(url, params=parameters, timeout=1)
             # logging.info(h.json())
 
             if h.status_code == 200:
@@ -120,61 +119,58 @@ def get_adcreatives(account_id,access_token,flag,adc_ids,dt):  # 获取创意
             else:
                 time.sleep(1)
                 logging.info("爬取失败 等待1s")
-
-
+        logging.info(f"{account_id}采集到创意")
         if 'data' in r.keys():
-            is_video=0
+            is_video = 0
             for i in r['data']['list']:
                 # logging.info(i)
 
-                if flag=='MP':
-                    if len(i['adcreative_elements'])>0:
+                if flag == 'MP':
+                    if len(i['adcreative_elements']) > 0:
                         d = i['adcreative_elements']
-                        title =d.get('title','')
+                        title = d.get('title', '')
                         description = d.get('description', '')
                         if 'image' in d.keys():
-                            image=d.get('image','')
+                            image = d.get('image', '')
                         elif 'image_list' in d.keys():
-                            image =','.join(d.get('image_list'))
+                            image = ','.join(d.get('image_list'))
                         elif 'video' in d.keys():
                             image = d['video']
-                            is_video=1
+                            is_video = 1
                         else:
                             image = ''
 
                     else:
-                        title = image=''
-
+                        title = image = ''
 
                     li.append((
-                        i['adcreative_id'],i['adcreative_name'],i['campaign_id'],image,title,
-                        i.get('promoted_object_type',''),i.get('page_type',''),
-                        i['page_spec'].get('page_id',''),i.get('promoted_object_id',''),
-                        '',description,'MP',account_id,dt,is_video
+                        i['adcreative_id'], i['adcreative_name'], i['campaign_id'], image, title,
+                        i.get('promoted_object_type', ''), i.get('page_type', ''),
+                        i['page_spec'].get('page_id', ''), i.get('promoted_object_id', ''),
+                        '', description, 'MP', account_id, dt, is_video
                     ))
                 else:
-                    if len(i['adcreative_elements'])>0:
-                        d =i['adcreative_elements']
+                    if len(i['adcreative_elements']) > 0:
+                        d = i['adcreative_elements']
                         if 'image' in d.keys():
-                            image =d['image']
+                            image = d['image']
                         elif 'element_story' in d.keys():
-                            image= ','.join([x['image'] for x in  d['element_story']])
+                            image = ','.join([x['image'] for x in d['element_story']])
                         else:
-                            image=''
-                        title =d.get('title','')
-                        description = d.get('description','')
+                            image = ''
+                        title = d.get('title', '')
+                        description = d.get('description', '')
 
 
                     else:
-                        image=title=description=''
-
+                        image = title = description = ''
 
                     li.append(
                         (
-                            i['adcreative_id'], i['adcreative_name'], i['campaign_id'],image,title,
+                            i['adcreative_id'], i['adcreative_name'], i['campaign_id'], image, title,
                             i.get('promoted_object_type', ''), i.get('page_type', ''),
                             i['page_spec'].get('page_id', ''), i.get('promoted_object_id', ''),
-                            ','.join(i['site_set']),description,'GDT',account_id,dt,is_video
+                            ','.join(i['site_set']), description, 'GDT', account_id, dt, is_video
 
                         )
                     )
@@ -186,24 +182,23 @@ def get_adcreatives(account_id,access_token,flag,adc_ids,dt):  # 获取创意
                 break
         else:
             break
+        logging.info(f"{account_id}创意分析结束")
     logging.info(f"{account_id}获取创意,结束")
-    if len(li)>0:
-        logging.info(f"{account_id}有创意："+str(len(li)))
-        sql='replace into adcreative_info values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) '
-        db.quchen_text.executeMany(sql,li)
-
+    if len(li) > 0:
+        logging.info(f"{account_id}有创意：" + str(len(li)))
+        sql = 'replace into adcreative_info values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) '
+        db.quchen_text.executeMany(sql, li)
 
 
-def images_info_get(account_id,access_token,image_ids):  # 获取图片信息
-    #接口https://developers.e.qq.com/docs/api/business_assets/image/images_get?version=1.3
-    fields = ('image_id','width','height','file_size','signature','preview_url')
+def images_info_get(account_id, access_token, image_ids):  # 获取图片信息
+    # 接口https://developers.e.qq.com/docs/api/business_assets/image/images_get?version=1.3
+    fields = ('image_id', 'width', 'height', 'file_size', 'signature', 'preview_url')
     interface = 'images/get'
     url = 'https://api.e.qq.com/v1.3/' + interface
 
     page = 1
     li = []
 
-
     while True:
 
         common_parameters = {
@@ -215,12 +210,12 @@ def images_info_get(account_id,access_token,image_ids):  # 获取图片信息
 
         parameters = {
             "account_id": account_id,
-            "filtering":[{
-                    "field": "image_id",
-                    "operator": "IN",
-                    "values": image_ids.split(',')
+            "filtering": [{
+                "field": "image_id",
+                "operator": "IN",
+                "values": image_ids.split(',')
 
-                    }],
+            }],
             "page": page,
             "page_size": 100
         }
@@ -243,7 +238,6 @@ def images_info_get(account_id,access_token,image_ids):  # 获取图片信息
         if 'data' in r.keys():
             li.extend(r['data']['list'])
 
-
         total_page = r['data']['page_info']['total_page']
         if total_page > page:
             page += 1
@@ -252,25 +246,24 @@ def images_info_get(account_id,access_token,image_ids):  # 获取图片信息
     # logging.info(li)
     data = []
     for i in li:
-        data.append((i['image_id'],i['width'],i['height'],i['signature'],i['preview_url']))
+        data.append((i['image_id'], i['width'], i['height'], i['signature'], i['preview_url']))
     # logging.info(data)
-    logging.info(f"{account_id} 有图片："+str(li.__len__()))
+    logging.info(f"{account_id} 有图片：" + str(li.__len__()))
     if li.__len__() > 0:
-        sql="replace into image_info value (%s,%s,%s,%s,%s)"
+        sql = "replace into image_info value (%s,%s,%s,%s,%s)"
         db.quchen_text.executeMany(sql, data)
         db.close()
 
 
-def video_info_get(account_id,access_token,image_ids):  # 获取视频信息
-    #接口https://developers.e.qq.com/docs/api/business_assets/video/videos_get?version=1.3
-    fields = ('video_id','width','height','file_size','signature','preview_url')
+def video_info_get(account_id, access_token, image_ids):  # 获取视频信息
+    # 接口https://developers.e.qq.com/docs/api/business_assets/video/videos_get?version=1.3
+    fields = ('video_id', 'width', 'height', 'file_size', 'signature', 'preview_url')
     interface = 'videos/get'
     url = 'https://api.e.qq.com/v1.3/' + interface
 
     page = 1
     li = []
 
-
     while True:
 
         common_parameters = {
@@ -282,12 +275,12 @@ def video_info_get(account_id,access_token,image_ids):  # 获取视频信息
 
         parameters = {
             "account_id": account_id,
-            "filtering":[{
-                    "field": "media_id",
-                    "operator": "IN",
-                    "values": image_ids.split(',')
+            "filtering": [{
+                "field": "media_id",
+                "operator": "IN",
+                "values": image_ids.split(',')
 
-                    }],
+            }],
             "page": page,
             "page_size": 100
         }
@@ -310,7 +303,6 @@ def video_info_get(account_id,access_token,image_ids):  # 获取视频信息
         if 'data' in r.keys():
             li.extend(r['data']['list'])
 
-
         total_page = r['data']['page_info']['total_page']
         if total_page > page:
             page += 1
@@ -319,11 +311,11 @@ def video_info_get(account_id,access_token,image_ids):  # 获取视频信息
     # logging.info(li)
     data = []
     for i in li:
-        data.append((i['video_id'],i['width'],i['height'],i['signature'],i['preview_url']))
+        data.append((i['video_id'], i['width'], i['height'], i['signature'], i['preview_url']))
     # logging.info(data)
-    logging.info(f"{account_id} 有视频："+ str(li.__len__()))
+    logging.info(f"{account_id} 有视频：" + str(li.__len__()))
     if li.__len__() > 0:
-        sql="replace into video_info value (%s,%s,%s,%s,%s)"
+        sql = "replace into video_info value (%s,%s,%s,%s,%s)"
         db.quchen_text.executeMany(sql, data)
         db.close()
 
@@ -335,28 +327,27 @@ def ad_info():
      select account_id,access_token,name channel,'MP' type from advertiser_vx where name !='' or name is not null 
      
      """)
-    total_data =[]
+    total_data = []
     executor = ThreadPoolExecutor(max_workers=max_workers)
     for i in accounts:
         # logging.info(i)
-        account_id =i[0]
+        account_id = i[0]
         access_token = i[1]
         type = i[3]
-        executor.submit(get_ad_info,account_id, access_token,type,total_data)
+        executor.submit(get_ad_info, account_id, access_token, type, total_data)
     executor.shutdown()
 
     logging.info(len(total_data))
-    if len(total_data)>0:
-        sql="replace into ad_info values(%s,%s,%s,%s,%s,%s,%s) "
+    if len(total_data) > 0:
+        sql = "replace into ad_info values(%s,%s,%s,%s,%s,%s,%s) "
         db.quchen_text.executeMany(sql, total_data)
 
 
 """获取广告基础信息"""
 
 
-def get_ad_info(account_id, access_token, flag,ad_ids,dt):
-
-    #接口为https://developers.e.qq.com/docs/apilist/ads/ad?version=1.3#a3
+def get_ad_info(account_id, access_token, flag, ad_ids, dt):
+    # 接口为https://developers.e.qq.com/docs/apilist/ads/ad?version=1.3#a3
     path = 'ads/get'
     fields = ('ad_id', 'ad_name', 'adcreative_id', 'adgroup_id', 'campaign_id')
     url = 'https://api.e.qq.com/v1.3/' + path
@@ -405,26 +396,25 @@ def get_ad_info(account_id, access_token, flag,ad_ids,dt):
                 li.append((str(i['ad_id']), i['ad_name'], i['adcreative_id'], i['campaign_id'], i['adgroup_id'],
                            account_id, flag, dt))
 
-
-    if li.__len__()>0:
-        logging.info(f"{account_id}有广告:"+str(li.__len__()))
+    if li.__len__() > 0:
+        logging.info(f"{account_id}有广告:" + str(li.__len__()))
         sql = "replace into ad_info values(%s,%s,%s,%s,%s,%s,%s,%s) "
         db.quchen_text.executeMany(sql, li)
     db.close()
 
 
-def get_ad_cost_day(account_id,access_token,flag,st,et):
-
+def get_ad_cost_day(account_id, access_token, flag, st, et):
     if flag == 'MP':
-        ad_cost_day_mp(account_id,access_token, st, et)
+        ad_cost_day_mp(account_id, access_token, st, et)
     else:
-        ad_cost_day_gdt(account_id,access_token, st, et)
+        ad_cost_day_gdt(account_id, access_token, st, et)
 
 
-def ad_cost_day_gdt(account_id,access_token,st,et):
-    #接口文档https://developers.e.qq.com/docs/api/insights/ad_insights/daily_reports_get?version=1.3
+def ad_cost_day_gdt(account_id, access_token, st, et):
+    # 接口文档https://developers.e.qq.com/docs/api/insights/ad_insights/daily_reports_get?version=1.3
     url = 'https://api.e.qq.com/v1.3/daily_reports/get'
-    fields = ('date', 'ad_id','adgroup_id', 'cost', 'view_count', 'ctr', 'follow_count','web_order_count','order_amount')
+    fields = (
+        'date', 'ad_id', 'adgroup_id', 'cost', 'view_count', 'ctr', 'follow_count', 'web_order_count', 'order_amount')
     li = []
     page = 1
     while True:
@@ -434,7 +424,7 @@ def ad_cost_day_gdt(account_id,access_token,st,et):
             'nonce': str(time.time()) + str(random.randint(0, 999999)),
             'fields': fields,
             "account_id": account_id,
-            "group_by" : ['ad_id','date'],
+            "group_by": ['ad_id', 'date'],
             "level": 'REPORT_LEVEL_AD',
             "page": page,
             "page_size": 1000,
@@ -449,21 +439,22 @@ def ad_cost_day_gdt(account_id,access_token,st,et):
                 parameters[k] = json.dumps(parameters[k])
 
         while True:
-            r  = requests.get(url, params=parameters).json()
-        # logging.info(r)
-            code =r['code']
-            if code==11017:
+            r = requests.get(url, params=parameters).json()
+            # logging.info(r)
+            code = r['code']
+            if code == 11017:
                 time.sleep(61)
             else:
                 break
 
         if r.get("data"):
             for i in r['data']['list']:
-                if i['cost']>0:
+                if i['cost'] > 0:
                     li.append(
                         (
-                            i['date'], i['ad_id'],i['adgroup_id'], i['cost']/100, i['view_count'], i['ctr']*i['view_count'],
-                            i['follow_count'],i['web_order_count'],i['order_amount']/100,account_id,'GDT'
+                            i['date'], i['ad_id'], i['adgroup_id'], i['cost'] / 100, i['view_count'],
+                            i['ctr'] * i['view_count'],
+                            i['follow_count'], i['web_order_count'], i['order_amount'] / 100, account_id, 'GDT'
                         )
                     )
 
@@ -479,9 +470,10 @@ def ad_cost_day_gdt(account_id,access_token,st,et):
         db.close()
 
 
-def ad_cost_day_mp(account_id,access_token,st,et):
+def ad_cost_day_mp(account_id, access_token, st, et):
     url = 'https://api.e.qq.com/v1.3/daily_reports/get'
-    fields = ('date', 'ad_id','adgroup_id', 'cost', 'view_count', 'valid_click_count', 'official_account_follow_count','order_count','order_amount')
+    fields = ('date', 'ad_id', 'adgroup_id', 'cost', 'view_count', 'valid_click_count', 'official_account_follow_count',
+              'order_count', 'order_amount')
     li = []
     page = 1
     while True:
@@ -518,17 +510,19 @@ def ad_cost_day_mp(account_id,access_token,st,et):
 
         if r.get("data"):
             for i in r['data']['list']:
-                if i['cost']>0:
+                if i['cost'] > 0:
                     li.append(
                         (
-                            i['date'],i['ad_id'],i['adgroup_id'],i['cost']/100,i['view_count'],i['valid_click_count'],
-                            i['official_account_follow_count'],i['order_count'],i['order_amount']/100,account_id,'MP'
+                            i['date'], i['ad_id'], i['adgroup_id'], i['cost'] / 100, i['view_count'],
+                            i['valid_click_count'],
+                            i['official_account_follow_count'], i['order_count'], i['order_amount'] / 100, account_id,
+                            'MP'
 
                         )
                     )
 
         total_page = r['data']['page_info']['total_page']
-        if page >=total_page:
+        if page >= total_page:
             break
         else:
             page += 1
@@ -539,6 +533,7 @@ def ad_cost_day_mp(account_id,access_token,st,et):
         db.quchen_text.executeMany('replace into ad_cost_day values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)', li)
         db.close()
 
+
 def daily_reports_get(access_token, account_id, level, start_date, end_date, fields):  # 获取wx投放计划日报数据
 
     interface = 'daily_reports/get'
@@ -635,7 +630,6 @@ def daily_qq_reports_get(access_token, account_id, compaign_id, level, start_dat
     return r.json()
 
 
-
 def mysql_insert_adcreative(data):
     db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com', 'superc', 'Cc719199895', 'quchen_text')
     cursor = db.cursor()
@@ -645,26 +639,18 @@ def mysql_insert_adcreative(data):
         cursor.executemany(sql, data)
         db.commit()
 
-        logging.info('insert [adcreative] '+ str(len(data)))
+        logging.info('insert [adcreative] ' + str(len(data)))
     except:
         db.rollback()
         logging.info('insert [adcreative] defeat')
 
 
-
-
-
 if __name__ == '__main__':
     account_id = 19206910
     access_token = '89079ccc8db047b078a0108e36a7e276'
     #
     account_id2 = 14709511
     access_token2 = 'e87f7b6f860eaeef086ddcc9c3614678'
-    get_ad_cost_day(account_id,access_token,'MP','2021-04-09','2021-04-09')
-
+    get_ad_cost_day(account_id, access_token, 'MP', '2021-04-09', '2021-04-09')
 
     # get_adcreatives(account_id,access_token,'MP','3187867673','2021-04-09')
-
-
-
-

app/api_data/tx_ad_cost/get_cost_older.py

@@ -53,9 +53,10 @@ def daily_reports_get(access_token, account_id, st, et, level, fields, err_num=0
             'access_token:{} code:{} message:{}'.format(str(access_token), str(r['code']), str(r['message'])))
         if err_num < 5:
             time.sleep(0.1)
-            return daily_reports_get(access_token, account_id, st, et, level, fields,err_num=err_num+1)
+            return daily_reports_get(access_token, account_id, st, et, level, fields, err_num=err_num + 1)
         DingTalkUtils().send(
-            '消耗日报请求出现问题\naccess_token:{} code:{} message:{}'.format(str(access_token), str(r['code']), str(r['message'])))
+            '消耗日报请求出现问题\naccess_token:{} code:{} message:{}'.format(str(access_token), str(r['code']),
+                                                                    str(r['message'])))
     return r