1

app/api_data/cost_util.py

@@ -64,7 +64,7 @@ def get_adcreatives(account_id,access_token,flag,adc_ids,dt):  # 获取创意
 
 
         if 'data' in r.keys():
-
+            is_video=0
             for i in r['data']['list']:
                 # print(i)
 
@@ -77,8 +77,11 @@ def get_adcreatives(account_id,access_token,flag,adc_ids,dt):  # 获取创意
                             image=d.get('image','')
                         elif 'image_list' in d.keys():
                             image =','.join(d.get('image_list'))
+                        elif 'short_video_struct' in d.keys():
+                            image = d['short_video_struct']["short_video1"]
+                            is_video=1
                         else:
-                            image=''
+                            image = ''
 
                     else:
                         title = image=''
@@ -88,7 +91,7 @@ def get_adcreatives(account_id,access_token,flag,adc_ids,dt):  # 获取创意
                         i['adcreative_id'],i['adcreative_name'],i['campaign_id'],image,title,
                         i.get('promoted_object_type',''),i.get('page_type',''),
                         i['page_spec'].get('page_id',''),i.get('promoted_object_id',''),
-                        '',description,'MP',account_id,dt
+                        '',description,'MP',account_id,dt,is_video
                     ))
                 else:
                     if len(i['adcreative_elements'])>0:
@@ -112,7 +115,7 @@ def get_adcreatives(account_id,access_token,flag,adc_ids,dt):  # 获取创意
                             i['adcreative_id'], i['adcreative_name'], i['campaign_id'],image,title,
                             i.get('promoted_object_type', ''), i.get('page_type', ''),
                             i['page_spec'].get('page_id', ''), i.get('promoted_object_id', ''),
-                            ','.join(i['site_set']),description,'GDT',account_id,dt
+                            ','.join(i['site_set']),description,'GDT',account_id,dt,is_video
 
                         )
                     )
@@ -130,7 +133,7 @@ def get_adcreatives(account_id,access_token,flag,adc_ids,dt):  # 获取创意
             break
     if len(li)>0:
         print(f"{account_id}有创意：",len(li))
-        sql='replace into adcreative_info values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) '
+        sql='replace into adcreative_info values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) '
         db.quchen_text.executeMany(sql,li)
 
 

app/api_data/get_cost.py

@@ -57,13 +57,14 @@ def adcreative(dt):
 
 	sql = f"""SELECT b.account_id,b.access_token,b.type,GROUP_CONCAT(adcreative_id)  from ad_info  a
 left join (
-select account_id,access_token,'MP' type  from advertiser_vx where (name !='' or name is not null) union 
+select account_id,access_token,'MP' type  from advertiser_vx where  (name !='' or name is not null) union 
 select account_id,access_token,'GDT' type from advertiser_qq where (name !='' or name is not null)
 ) b on a.account_id=b.account_id
 where a.dt='{dt}'
-GROUP BY b.account_id,b.access_token,b.type"""
+GROUP BY b.account_id,b.access_token,b.type having b.account_id is not null """
 
 	accounts = db.quchen_text.getData(sql)
+	# print(accounts)
 	executor = ThreadPoolExecutor(max_workers=max_workers)
 	for account in accounts:
 		executor.submit(get_adcreatives, account[0], account[1],account[2],account[3],dt)
@@ -119,11 +120,12 @@ if __name__ == '__main__':
 	# run()
 	# ad_cost_day('2020-04-08','2021-04-07')
 	
-	day()
+	# day()
 	#
 
 	# day()
-	# run('2021-04-05')
+	# run('2021-05-10')
+	adcreative('2021-05-11')
 	# for dt in list(reversed(du.getDateLists('2020-04-08','2021-04-09'))):
 	# 	print(dt)
 	# 	ad(dt)

app/api_data/get_order.py

@@ -173,8 +173,21 @@ def daily():
     youshuge(st, et)
     yangguang(st, et)
 
+def yestoday():
+    st = et = du.get_n_days(-1)
+    huasheng(st, et)
+    qiyue(st, et)
+    qiyueyousheng(st, et)
+    wending(st, et)
+    zhangdu(st, et)
+    zhangzhongyun(st, et)
+    yueweng(st, et)
+    youshuge(st, et)
+    yangguang(st, et)
+
 
 if __name__ == '__main__':
     # zhangzhongyun()
     # yangguang()
-    huasheng("2021-04-12",'2021-05-10')
+    # huasheng("2021-04-12",'2021-05-10')
+    yueweng("2021-05-11", '2021-05-11')

app/daily08.py

@@ -0,0 +1,17 @@
+from app.api_data import get_order
+from app.etl import data_stat_run
+from model.DateUtils import DateUtils
+from app.api_data import get_cost
+from app.etl.dw import dw_image_cost_day
+du = DateUtils()
+
+
+
+if __name__ == '__main__':
+
+    thedate = du.get_n_days(-1)
+    # 订单数据
+    get_order.yestoday()
+
+    # 数据处理
+    data_stat_run.do_order(thedate, thedate)

app/dailyRun.py

@@ -16,9 +16,6 @@ if __name__ == '__main__':
     # 数据处理
     data_stat_run.daily()
 
-    # 阳光订单
-    get_order.daily_yg()
-
     # 广告数据
     get_cost.day()
     dw_image_cost_day.day()

app/etl/MaterialLibrary/ddd.py

@@ -0,0 +1,89 @@
+"""
+desc : 素材库数据清洗
+每日运行一次
+
+"""
+from model.DataBaseUtils import MysqlUtils
+db =MysqlUtils()
+
+def title():
+    sql = """select REPLACE(REPLACE(title, CHAR(10), ''), CHAR(13), '') content,
+        1 type,
+        0 create_by,
+        sum(click_count) click_times,
+        sum(view_count) view_times,
+        group_concat(distinct book) novels,
+        max(dt) end_date,min(dt) start_date
+        from dw_image_cost_day where title!='' and title is not null  GROUP BY REPLACE(REPLACE(title, CHAR(10), ''), CHAR(13), '')  limit 100
+				"""
+
+    df =  db.dm.getData_pd(sql)
+    print(df)
+
+    key = ["content", "type"]
+    tag = ["view_times", "click_times", "novels", "start_date", "end_date", "create_by"]
+    table = "t_ads_content"
+
+    db.zx_test.dfsave2mysql(df, table, key, tag)
+
+def description():
+    sql = """select REPLACE(REPLACE(description, CHAR(10), ''), CHAR(13), '') content,
+            2 type,
+            0 create_by,
+            sum(click_count) click_times,
+            sum(view_count) view_times,
+            group_concat(distinct book) novels,
+            max(dt) end_date,min(dt) start_date
+            from dw_image_cost_day where description!='' and description is not null  GROUP BY REPLACE(REPLACE(description, CHAR(10), ''), CHAR(13), '')  limit 100
+    				"""
+
+    df = db.dm.getData_pd(sql)
+    print(df)
+
+    key = ["content", "type"]
+    tag = ["view_times", "click_times", "novels", "start_date", "end_date", "create_by"]
+    table = "t_ads_content"
+
+    db.zx_test.dfsave2mysql(df, table, key, tag)
+
+
+def image():
+    sql="""select signature,
+            sum(click_count) click_times,
+            sum(view_count) view_times,
+            group_concat(distinct book) novels ,
+            max(dt) end_date,
+            min(dt) start_date,
+            min(preview_url) content,
+            1 type,
+            if(locate(',',signature)>0,0,1) single_img,
+            0 create_by  
+            from dw_image_cost_day where dt='2021-05-10'   GROUP BY  signature"""
+
+    df = db.dm.getData_pd(sql)
+    print(df)
+
+    key = ["signature"]
+    tag = ["view_times", "click_times", "novels", "start_date", "end_date", "create_by", "single_img", "content"]
+    table = "t_ads_media"
+
+    db.zx_test.dfsave2mysql(df, table, key, tag)
+
+
+def adcreative():
+    sql="""select signature,title,description,
+sum(click_count) click_count,
+sum(view_count) view_count,
+group_concat(distinct book),
+max(dt),min(dt),
+min(preview_url)
+
+from dw_image_cost_day where dt='2021-05-10' and signature!=''   GROUP BY  signature,title,description
+				"""
+
+
+if __name__ == '__main__':
+    # title()
+    # description()
+    image()
+

app/etl/data_stat_run.py

@@ -59,4 +59,4 @@ if __name__ == '__main__':
     # hourly()
     # do_cost('2021-02-10','2021-02-10')
 
-    do_order('2021-05-07','2021-05-07')
+    do_order('2021-05-11','2021-05-11')

app/etl/dw/dw_image_cost_day.py

@@ -76,8 +76,8 @@ def day():
 
 
 if __name__ == '__main__':
-    run('2021-05-10')
+    # run('2021-05-01')
 
-    # for i in du.getDateLists('2021-04-01','2021-04-13'):
-    #     print(i)
-    #     run(i)
+    for i in du.getDateLists('2021-05-01','2021-05-09'):
+        print(i)
+        run(i)