1

app/api_data/get_cost.py

@@ -116,6 +116,7 @@ def campaign(dt):
 	executor.shutdown()
 
 
+
 def run(dt):
 	"""
 		1.拉取有消耗的广告
@@ -160,10 +161,11 @@ if __name__ == '__main__':
 	# run('2021-05-10')
 	# adcreative('2021-05-11')
 	# video('2021-05-14')
-	campaign('2021-05-14')
-	# for dt in list(reversed(du.getDateLists('2020-04-08','2021-04-09'))):
-	# 	print(dt)
-	# 	ad(dt)
-	# 	adcreative(dt)
-	# 	image(dt)
+	# campaign('2021-05-14')
+	for dt in list(reversed(du.getDateLists('2020-05-18','2021-05-17'))):
+		print(dt)
+		# ad(dt)
+		adcreative(dt)
+		# image(dt)
+		# video(dt)
 

app/api_data/tx_ad_cost/TxCostUtils.py

@@ -0,0 +1,16 @@
+
+from app.api_data.cost_util import *
+from model.DateUtils import DateUtils
+from model.DataBaseUtils import MysqlUtils
+du = DateUtils()
+
+def get_accounts(filter=None):
+    db = MysqlUtils()
+    if filter:
+		if filter=='MP':
+			return db.quchen_text.getData("select account_id,access_token,name channel from advertiser_vx where (name !='' or name is not null)")
+		else:
+			return db.quchen_text.getData("select account_id,access_token,name channel from advertiser_qq where (name !='' or name is not null)")
+
+    return db.quchen_text.getData("select account_id,access_token,name channel,'MP' flag from advertiser_vx where (name !='' or name is not null) union "
+								   "select account_id,access_token,name channel,'GDT' flag from advertiser_qq where (name !='' or name is not null)")

app/api_data/tx_ad_cost/campaign_cost_daily.py

@@ -0,0 +1,22 @@
+"""
+计划维度日报"""
+
+from model.DataBaseUtils import MysqlUtils
+from concurrent.futures import ThreadPoolExecutor
+from .TxCostUtils import get_accounts
+
+
+def run(st, et):
+    executor = ThreadPoolExecutor(max_workers=10)
+    for account in get_accounts():
+	    executor.submit(get_campaign_cost_daily, account[0], account[1], account[3], st, et)
+    executor.shutdown()
+
+
+
+def get_campaign_cost_daily(account_id, token, st, et):
+    """1,获取计划维度日报，把有消耗的计划入库
+    2，根据有消耗的计划，拉取计划的基础信息入库
+    """
+
+

app/etl/MaterialLibrary/ddd.py → app/etl/MaterialLibrary/MaterialDataClean.py

@@ -8,40 +8,44 @@ db =MysqlUtils()
 
 def title():
     sql = """select REPLACE(REPLACE(title, CHAR(10), ''), CHAR(13), '') content,
-        1 type,
-        0 create_by,
+        sum(cost) consume_amount,
         sum(click_count) click_times,
         sum(view_count) view_times,
         group_concat(distinct book) novels,
         max(dt) end_date,min(dt) start_date
-        from dw_image_cost_day where title!='' and title is not null  GROUP BY REPLACE(REPLACE(title, CHAR(10), ''), CHAR(13), '')  limit 100
+        from dw_image_cost_day where title!='' and title is not null  GROUP BY REPLACE(REPLACE(title, CHAR(10), ''), CHAR(13), '')  limit 1000
 				"""
 
     df =  db.dm.getData_pd(sql)
     print(df)
+    df["data_type"] = 'all'
+    df['type'] = 1
+    df['create_by'] = 0
 
     key = ["content", "type"]
-    tag = ["view_times", "click_times", "novels", "start_date", "end_date", "create_by"]
+    tag = ["view_times", "click_times", "novels", "start_date", "end_date", "create_by",'data_type', 'consume_amount']
     table = "t_ads_content"
 
     db.zx_test.dfsave2mysql(df, table, key, tag)
 
 def description():
     sql = """select REPLACE(REPLACE(description, CHAR(10), ''), CHAR(13), '') content,
-            2 type,
-            0 create_by,
+            sum(cost) consume_amount,
             sum(click_count) click_times,
             sum(view_count) view_times,
             group_concat(distinct book) novels,
             max(dt) end_date,min(dt) start_date
-            from dw_image_cost_day where description!='' and description is not null  GROUP BY REPLACE(REPLACE(description, CHAR(10), ''), CHAR(13), '')  limit 100
+            from dw_image_cost_day where description!='' and description is not null  GROUP BY REPLACE(REPLACE(description, CHAR(10), ''), CHAR(13), '')  limit 1000
     				"""
 
     df = db.dm.getData_pd(sql)
     print(df)
 
+    df["data_type"] = 'all'
+    df['type'] = 2
+    df['create_by'] = 0
     key = ["content", "type"]
-    tag = ["view_times", "click_times", "novels", "start_date", "end_date", "create_by"]
+    tag = ["view_times", "click_times", "novels", "start_date", "end_date", "create_by",'data_type', 'consume_amount']
     table = "t_ads_content"
 
     db.zx_test.dfsave2mysql(df, table, key, tag)
@@ -49,41 +53,56 @@ def description():
 
 def image():
     sql="""select signature,
+            sum(cost) consume_amount,
             sum(click_count) click_times,
             sum(view_count) view_times,
             group_concat(distinct book) novels ,
             max(dt) end_date,
             min(dt) start_date,
             min(preview_url) content,
-            1 type,
-            if(locate(',',signature)>0,0,1) single_img,
-            0 create_by  
-            from dw_image_cost_day where dt='2021-05-10'   GROUP BY  signature"""
+            if(flag=1,2,1) type,  
+            if(locate(',',signature)>0,0,1) single_img
+            from dw_image_cost_day  where signature is not null and signature !=''  GROUP BY  signature,flag limit 1000"""
 
     df = db.dm.getData_pd(sql)
-    print(df)
+    # print(df)
+
+    df['create_by'] = 0
+    df["data_type"] = 'all'
 
     key = ["signature"]
-    tag = ["view_times", "click_times", "novels", "start_date", "end_date", "create_by", "single_img", "content"]
+    tag = ["view_times", "click_times", "novels", "start_date", "end_date", "create_by", "single_img", "content",'consume_amount','type']
     table = "t_ads_media"
 
     db.zx_test.dfsave2mysql(df, table, key, tag)
 
 
 def adcreative():
-    sql="""select signature,title,description,
-sum(click_count) click_count,
-sum(view_count) view_count,
-group_concat(distinct book),
-max(dt),min(dt),
-min(preview_url)
-
-from dw_image_cost_day where dt='2021-05-10' and signature!=''   GROUP BY  signature,title,description
+    sql="""select signature,title,description article,
+sum(click_count) click_times,
+sum(view_count) view_times,
+sum(cost) consume_amount,
+group_concat(distinct book) novels,
+max(dt) start_date,min(dt) end_date,
+min(preview_url) media,
+type channel,
+if(flag=1,2,1) type,
+if(locate(',',signature)>0,0,1) single_img
+
+from dw_image_cost_day where signature is not null and signature!=''   GROUP BY  signature,title,description,type,flag
 				"""
 
+    df = db.dm.getData_pd(sql)
+
+    key = ["signature",'title','article']
+    tag = ["view_times", "click_times", "novels", "start_date", "end_date","type","channel",'consume_amount','single_img','media']
+    table = "t_ads_idea"
+
+    db.zx_test.dfsave2mysql(df, table, key, tag)
+
 
 if __name__ == '__main__':
     # title()
     # description()
-    image()
-
+    # image()
+    adcreative()

app/etl/dw/dw_image_cost_day.py

@@ -18,7 +18,7 @@ def run(dt):
             left join adcreative_info c on b.adcreative_id=c.adcreative_id
             left join channel_by_account_daily e on b.account_id=e.account_id and a.dt=e.dt
             left join channel_info_daily f on e.channel=f.channel and e.dt=f.dt
-            where a.dt='{dt}'  
+            where a.dt='{dt}'  and c.is_video=0
             group by a.dt,b.type,title,description,book,platform,stage,image_id,e.channel,pitcher  
             
             """
@@ -59,6 +59,28 @@ def run(dt):
     # print(data)
     # print(data)
     # exit(0)
+    sql_video = f"""SELECT a.dt,b.type,sum(a.cost),sum(view_count),sum(click_count),sum(follow_count),sum(order_count),sum(order_amount),
+            title,description,book,platform,stage,e.channel,pitcher,ifnull(image_id,''),g.signature,g.preview_url,1
+            from 
+            ad_cost_day a 
+            left join ad_info b on a.ad_id=b.ad_id
+            left join adcreative_info c on b.adcreative_id=c.adcreative_id
+            left join channel_by_account_daily e on b.account_id=e.account_id and a.dt=e.dt
+            left join channel_info_daily f on e.channel=f.channel and e.dt=f.dt
+            left join video_info g on c.image_id=g.video_id
+            where a.dt='{dt}'  and c.is_video=1
+            group by a.dt,b.type,title,description,book,platform,stage,image_id,e.channel,pitcher """
+
+
+    data_video = db.quchen_text.get_data_list(sql_video)
+    data.extend(data_video)
+
+
+
+    # print(data)
+
+
+
     db.dm.execute(f'delete from dw_image_cost_day where dt="{dt}"')
     db.dm.executeMany("replace into dw_image_cost_day values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",data)
 
@@ -77,7 +99,7 @@ def day():
 
 
 if __name__ == '__main__':
-    run('2021-05-17')
+    run('2021-05-18')
 
     # for i in du.getDateLists('2021-05-01','2021-05-09'):
     #     print(i)