广告计划拉取优化

dgp/tests/get_campaign_daily.py

@@ -16,7 +16,7 @@ import token_list as tl
 from concurrent.futures import ThreadPoolExecutor
 from util import date_util
 
-max_workers=5
+
 
 def adcreatives_get(access_token,account_id,fields) : #获取创意
 
@@ -43,7 +43,16 @@ def adcreatives_get(access_token,account_id,fields) : #获取创意
         if type(parameters[k]) is not str:
             parameters[k] = json.dumps(parameters[k])
 
-    r = requests.get(url, params = parameters).json()
+    while True:
+        h = requests.get(url, params=parameters)
+        if h.status_code==200:
+            r=h.json()
+            break
+        else:
+            time.sleep(1)
+            print("爬取失败 等待1s")
+
+
     if 'data' in r.keys():
         list1 = list1+r['data']['list']
 
@@ -72,7 +81,6 @@ def adcreatives_get(access_token,account_id,fields) : #获取创意
                     list1 = list1+r['data']['list']            
     return list1
 
-#print(adcreatives_get('3bbbae77bed9fcde94cc0f1742a18c6e',11436446,('campaign_id','adcreative_id','adcreative_name','adcreative_elements','promoted_object_type','page_type','page_spec','link_page_spec','universal_link_url','promoted_object_id')))
 
 
 def ads_get(access_token,account_id,fields) : #获取广告
@@ -103,7 +111,7 @@ def ads_get(access_token,account_id,fields) : #获取广告
 
     return r.json()
 
-#print(ads_get('2a674bef201314d338be30420369671f',14985162,('ad_id','ad_name','adcreative_id','adcreative')))
+
     
 
 def wechat_pages_get(access_token,account_id,page_id,fields) : #获取微信原生页
@@ -177,12 +185,13 @@ def adgroups_get(access_token,account_id,fields) : #获取广告组
 
 #print(adgroups_get('2a674bef201314d338be30420369671f',14985162,('campaign_id','adgroup_id','adgroup_name','optimization_goal','billing_event','bid_amount','daily_budget','targeting','begin_date','end_date','time_series','bid_strategy','cold_start_audience','auto_audience','expand_enabled','expand_targeting','deep_conversion_spec','deep_optimization_action_type','conversion_id','deep_conversion_behavior_bid','deep_conversion_worth_rate','system_status')))
 
-def images_get(access_token,account_id,fields) : #获取图片信息
+def images_get(access_token,account_id,li) : #获取图片信息
 
     import json
     import random
     import requests
     import time
+    fields=('image_id','preview_url')
 
     interface = 'images/get'
     url = 'https://api.e.qq.com/v1.1/' + interface
@@ -207,7 +216,16 @@ def images_get(access_token,account_id,fields) : #获取图片信息
         if type(parameters[k]) is not str:
             parameters[k] = json.dumps(parameters[k])
 
-    r = requests.get(url, params = parameters).json()
+    while True:
+        h = requests.get(url, params = parameters)
+        if h.status_code==200:
+            r=h.json()
+            break
+        else:
+            time.sleep(1)
+            print("请求出错 等待1s..")
+
+
     if 'data' in r.keys():
         list1 = list1+r['data']['list']
 
@@ -215,16 +233,14 @@ def images_get(access_token,account_id,fields) : #获取图片信息
         if total_page>1:
             for page in range(2,total_page+1):
                 common_parameters = {
-            'access_token': access_token, 
-            'timestamp': int(time.time()), 
-            'nonce': str(time.time()) + str(random.randint(0, 999999)),
-            'fields':fields
-        }
+                'access_token': access_token,
+                'timestamp': int(time.time()),
+                'nonce': str(time.time()) + str(random.randint(0, 999999)),
+                'fields':fields}
                 parameters = {
-            "account_id": account_id,
-            "page": page,
-            "page_size": 100
-        }   
+                "account_id": account_id,
+                "page": page,
+                "page_size": 100}
                 parameters.update(common_parameters)
                 for k in parameters:
                     if type(parameters[k]) is not str:
@@ -232,11 +248,14 @@ def images_get(access_token,account_id,fields) : #获取图片信息
 
                 r = requests.get(url, params = parameters).json()
                 if 'data' in r.keys():
-                    list1 = list1+r['data']['list']            
-    return list1  
+                    list1 = list1+r['data']['list']
+    for x in list1:
+        x["account_id"]=account_id
+        li.append(tuple(x.values()))
+
+
 
 
-#print(images_get('2a674bef201314d338be30420369671f',14985162,('image_id','preview_url')))
 
 def campaigns_get(access_token,account_id,fields) : #获取推广计划
 
@@ -398,80 +417,52 @@ def daily_qq_reports_get(access_token,account_id,compaign_id,level,start_date,en
 
 def mysql_insert_daily_vx_campaign(data):
     db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com','superc','Cc719199895','quchen_text')
-    cursor = db.cursor() 
-    time1 = time.time()
-    sql = 'insert ignore into daily_vx_campaign (account_id,date,campaign_id,view_count,cost,ctr,cpc,order_roi,thousand_display_price,valid_click_count,official_account_follow_count,conversions_count,official_account_follow_rate,conversions_rate,order_count,order_rate,order_unit_price,first_day_order_amount) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
-    try:
-        cursor.executemany(sql,data)
-        db.commit()
-        cost_time =round((time.time()-time1)/60,1)
-        print('insert_daily_vx_campaign access',len(data),'cost_minutes:',cost_time)
-    except:
-        db.rollback()
-        print('insert_daily_vx_campaign defeat')
-        
-def mysql_insert_daily_qq_campaign(data):
-    db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com','superc','Cc719199895','quchen_text')
-    cursor = db.cursor() 
-    time1 = time.time()
-    sql = 'insert ignore into daily_qq_campaign (account_id,date,campaign_id,view_count,thousand_display_price,valid_click_count,ctr,cpc,cost,order_roi) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
+    cursor = db.cursor()
+    sql = 'replace into daily_vx_campaign (account_id,date,campaign_id,view_count,cost,ctr,cpc,order_roi,thousand_display_price,valid_click_count,official_account_follow_count,conversions_count,official_account_follow_rate,conversions_rate,order_count,order_rate,order_unit_price,first_day_order_amount) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
     try:
         cursor.executemany(sql,data)
         db.commit()
-        cost_time =round((time.time()-time1)/60,1)
-        print('insert_daily_qq_campaign access',len(data),'cost_minutes:',cost_time)
+        print('insert [daily_vx_campaign]',len(data))
     except:
         db.rollback()
-        print('insert_daily_qq_campaign defeat')
+        print('insert [daily_vx_campaign] defeat')
 
-def mysql_insert_campaign_vx(data,data2):
+def mysql_insert_campaign_vx(data):
     db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com','superc','Cc719199895','quchen_text')
-    cursor = db.cursor() 
-    time1 = time.time()
-    sql = 'insert ignore into campaign_vx (campaign_id,campaign_name,configured_status,campaign_type,promoted_object_type,daily_budget,created_time,last_modified_time,account_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s)'
-    sql2 = 'delete from campaign_vx where campaign_id =%s '
-    try:
-        cursor.executemany(sql2,data2)
-        db.commit()
-        print('delete campaign_vx access',len(data2))
-    except:
-        db.rollback()
-        print('delete campaign_vx defeat')
+    cursor = db.cursor()
+    sql = 'replace into campaign_vx (campaign_id,campaign_name,configured_status,campaign_type,promoted_object_type,daily_budget,created_time,last_modified_time,account_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s)'
+
     try:
         cursor.executemany(sql,data)
         db.commit()
-        cost_time =round((time.time()-time1)/60,1)
-        print('insert_campaign_vx access',len(data),'cost_minutes:',cost_time)
+        print('insert [campaign_vx] ',len(data))
     except:
         db.rollback()
-        print('insert_campaign_vx defeat')
+        print('insert [campaign_vx] defeat')
 
 
 def mysql_insert_adcreative(data):
     db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com','superc','Cc719199895','quchen_text')
     cursor = db.cursor() 
-    time1 = time.time()
-    sql = 'insert ignore into adcreative (campaign_id,adcreative_id,adcreative_name,image_id,title,promoted_object_type,page_type,page_id,link_page_id,promoted_object_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
+
+    sql = 'replace into adcreative (campaign_id,adcreative_id,adcreative_name,image_id,title,promoted_object_type,page_type,page_id,link_page_id,promoted_object_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
     try:
         cursor.executemany(sql,data)
         db.commit()
-        cost_time =round((time.time()-time1)/60,1)
-        print('insert_adcreative access',len(data),'cost_minutes:',cost_time)
+
+        print('insert [adcreative] ',len(data))
     except:
         db.rollback()
-        print('insert_adcreative defeat')
+        print('insert [adcreative] defeat')
 
 
 def mysql_insert_image(data):
     db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com','superc','Cc719199895','quchen_text')
-    cursor = db.cursor() 
-    time1 = time.time()
-    sql = 'insert ignore into image (image_id,preview_url,account_id) values (%s,%s,%s)'
+    cursor = db.cursor()
+    sql = 'replace into image (image_id,preview_url,account_id) values (%s,%s,%s)'
     try:
         cursor.executemany(sql,data)
         db.commit()
-        cost_time =round((time.time()-time1)/60,1)
-        print('insert image access',len(data),'cost_minutes:',cost_time)
     except:
         db.rollback()
         print('insert image defeat')
@@ -482,6 +473,7 @@ def v_data(x,st,et,q,r,p):
     start_date = time.strftime("%Y-%m-%d", time.localtime(st))
     end_date = time.strftime("%Y-%m-%d", time.localtime(et))
 
+    # 获取推广计划
     l = campaigns_get(access_token, account_id, (
     'campaign_id', 'campaign_name', 'configured_status', 'campaign_type', 'promoted_object_type', 'daily_budget',
     'budget_reach_date', 'created_time', 'last_modified_time', 'speed_mode', 'is_deleted'))
@@ -507,21 +499,20 @@ def v_data(x,st,et,q,r,p):
 
 
 
+
 def get_daily_vx_campaign(st,et): #获取投放计划、日报数据
     token_list_v = tl.token_list_vx
     r = []
     p = []
     q=[]
+    t1=time.time()
     executor = ThreadPoolExecutor(max_workers=max_workers)
     for x in token_list_v:
         executor.submit(v_data,x,st,et,q,r,p)
     executor.shutdown()
-
+    print("获取投放日报数据 {} 广告推广{} cost:{}s".format(len(r),len(p),int(time.time()-t1)))
     mysql_insert_daily_vx_campaign(r)
-    mysql_insert_campaign_vx(p,q)
-
-
-    
+    mysql_insert_campaign_vx(p)
 
 
 
@@ -533,23 +524,23 @@ def get_daily_qq_campaign(st,et):
         access_token = x[2]
         start_date = st
         end_date = et
-        
+
         l = campaigns_get(access_token,account_id,('campaign_id','campaign_name','configured_status','campaign_type','promoted_object_type','daily_budget','budget_reach_date','created_time','last_modified_time','speed_mode','is_deleted'))
 
         for ll in l:
             campaign_id =ll['campaign_id']
-            
+
             data_list = daily_qq_reports_get(access_token,account_id,campaign_id,'REPORT_LEVEL_CAMPAIGN',start_date,end_date,('account_id','date','campaign_id','view_count','thousand_display_price','valid_click_count','ctr','cpc','cost','order_roi'))
             if len(data_list['data']['list'])>0:
                 print(data_list)
                 # print(l)
-            
+
             if 'data' in data_list.keys():
-            
+
                 for y in data_list['data']['list']:
-                    
-                    y = tuple(y.values()) 
-                    r=r+((y),) 
+
+                    y = tuple(y.values())
+                    r=r+((y),)
 
 
 def get_campaign_update_list():
@@ -561,10 +552,10 @@ def get_campaign_update_list():
         cursor.execute(sql,data)
         db.commit()
         x=cursor.fetchall()
-        # print('access get campaign update list',x)
+
     except:
         db.rollback()
-        # print('defeat get campaign update list')
+
     a = []
     if len(x)>0:
         for t in x:
@@ -574,10 +565,10 @@ def get_campaign_update_list():
         cursor.executemany(sql2,a)
         db.commit()
         y=cursor.fetchall()
-        # print('access delete adcreative',y)
+
     except:
         db.rollback()
-        # print('defeat delete adcreative')
+
     return x
 
 
@@ -656,43 +647,48 @@ def adcreative_vx_data(x,r):
 def get_adcreative_vx():
     token_list_vx=get_campaign_update_list()
     r = []
+    t1 = time.time()
     executor = ThreadPoolExecutor(max_workers=max_workers)
     for x in token_list_vx:
         executor.submit(adcreative_vx_data,x,r)
     executor.shutdown()
+    print("获取广告创意{}cost:{}s".format(len(r),int(time.time()-t1)))
     mysql_insert_adcreative(r)
 
 
 def get_image_imformation():
     token_list_vx = tl.token_list_vx
-    r = ()
+    li = []
+    t1=time.time()
+    executor = ThreadPoolExecutor(max_workers=max_workers)
     for x in token_list_vx:
         account_id = x[0]
         access_token = x[2]
-        
-        l = images_get(access_token,account_id,('image_id','preview_url'))
-        if len(l)>0:
-            for ll in l:
-                ll['account_id']=account_id
-                y=tuple(ll.values())
-                r=r+((y),)
-    mysql_insert_image(r)
+        executor.submit(images_get,access_token,account_id,li)
+    executor.shutdown()
+    print("获取图片信息{}条 cost:{}s".format(len(li),int(time.time()-t1)))
+    mysql_insert_image(li)
 
 def start_all_job():
     start_time = date_util.get_n_day(n=-1,is_timestamp=1)
     end_time = date_util.get_n_day(n=-1,is_timestamp=1)
 
     print("run[{0}] data".format(date_util.stamp_to_str(start_time)[:10]))
-
+    print("线程数:",max_workers)
+    print("开始获取计划日报----------")
     get_daily_vx_campaign(start_time,end_time)
-    print("get_daily_vx success----------")
+
+    print("开始获取广告创意信息--------------")
     get_adcreative_vx()
-    print("get_adcreative success--------------")
+
+    print("开始获取图片信息 ")
     get_image_imformation()
 
+    print("success")
 
 if __name__ == '__main__':
-    print("============start at " + str(datetime.today()) + "===================")
+    print("============start at " + str(datetime.today())[:19] + "===================")
+    max_workers = 100
     start_all_job()