ADD:添加日志数据&添加线程监控

app/api_data/platform_order/yangguang.py

@@ -240,7 +240,7 @@ def daily_yg():
 
 if __name__ == '__main__':
 
-    # get_channel_info()
+    get_channel_info()
     # exit(0)
     # yangguang('2020-09-20','2020-10-01')
     # # exit(0)
@@ -253,4 +253,3 @@ if __name__ == '__main__':
     # parse_order_data()
     # get_channel_info()
     yangguang()
-

app/api_data/tx_ad_cost/get_cost.py

@@ -4,6 +4,8 @@ from model.DateUtils import DateUtils
 from model.DataBaseUtils import MysqlUtils
 from concurrent.futures import ThreadPoolExecutor
 from model.DingTalkUtils import DingTalkUtils
+import logging
+
 db = MysqlUtils()
 du = DateUtils()
 
@@ -125,12 +127,21 @@ def run(dt):
 		4.用创意信息中的图片id 去获取图片的基础信息
 		"""
 	try:
-
+		logging.info('获取广告id,开始')
 		ad_cost_day(dt, dt)
+		logging.info('获取广告id,结束')
+		logging.info('获取广告基础信息,开始')
 		ad(dt)
+		logging.info('获取广告基础信息,结束')
+		logging.info('获取广告创意素材,开始')
 		adcreative(dt)
+		logging.info('获取广告创意素材,结束')
+		logging.info('获取图片信息,开始')
 		image(dt)
+		logging.info('获取图片信息,结束')
+		logging.info('获取视频信息,开始')
 		video(dt)
+		logging.info('获取视频信息,结束')
 
 	except:
 		DingTalkUtils.send("拉取广告数据出错")

app/api_data/tx_ad_cost/get_cost_older.py

@@ -159,8 +159,11 @@ def old_cost_daily():
 if __name__ == '__main__':
 
     # run()
-    old_cost_daily()
-
+    # old_cost_daily()
+    st = du.get_n_days(-6)
+    et = du.get_n_days(-1)
+    print(st,et)
+    run(st, et)
 
 
 

app/crontab_task/ad_hourly.py

@@ -1,23 +1,51 @@
 from model.DateUtils import DateUtils
 from app.api_data.tx_ad_cost import get_cost
 from app.etl.dw import dw_image_cost_day
-du = DateUtils()
 import time
 from model.DingTalkUtils import DingTalkUtils
+import logging
+from logging import handlers
+import threading
 
+du = DateUtils()
 
-if __name__ == '__main__':
+def hourly_run():
+    # 广告数据
+    get_cost.hourly()# 广告相关消耗数据
+    dw_image_cost_day.hourly()
 
+if __name__ == '__main__':
+    logging.basicConfig(
+        handlers=[
+            logging.handlers.RotatingFileHandler('./ad_hourly.log',
+                                                 maxBytes=10 * 1024 * 1024,
+                                                 backupCount=5,
+                                                 encoding='utf-8')
+            , logging.StreamHandler()  # 供输出使用
+        ],
+        level=logging.INFO,
+        format="%(asctime)s - %(levelname)s %(filename)s %(funcName)s %(lineno)s - %(message)s"
+    )
+
+    logging.info('广告素材任务,开始')
     st = time.time()
 
+    hourly_thread=threading.Thread(target=hourly_run)
+    hourly_thread.daemon=True
+    hourly_thread.start()
 
-    # 广告数据
-    get_cost.hourly()  # 广告相关消耗数据
-    dw_image_cost_day.hourly()
+    while 1:
+        if time.time()-st > 3000:
+            logging.info('线程运行超过50分钟,强制停止')
+            break
+        if not hourly_thread.is_alive():
+            break
 
     if int(time.time()-st)>1500:
         DingTalkUtils.send(f"小时任务耗时{int(time.time()-st)}秒",phone="15168342316")
 
+    logging.info('广告素材任务,结束')
+
 
 
 

app/etl/data_stat_task.py

@@ -145,8 +145,6 @@ def channel_info_daily(ymd):
 
 
 
-
-
     insert_sql="replace into channel_info_daily values (%s,%s,%s,%s,%s,%s) "
 
     db.quchen_text.executeMany(insert_sql,data)

app/etl/dw/dw_image_cost_day.py

@@ -35,6 +35,7 @@ def run(dt):
         # print(i)
         li.extend(i[-1].split(','))
     # print(li)
+    #TODO:之后如果一天产生的图片过多,可能超过sql的字符限制
 
     sql3 = f"select image_id,preview_url,signature from image_info where  image_id in ({str(set(li))[1:-1]})"
 
@@ -87,15 +88,18 @@ def run(dt):
 
 def hourly():
     try:
+        logging.info('广告数据清洗,开始')
         run(du.getNow())
+        logging.info('广告数据清洗,结束')
     except:
         DingTalkUtils.send("广告数据清洗失败")
 
 def day():
+    logging.info('广告数据清洗,开始')
     for i in du.getDateLists(du.get_n_days(-10), du.get_n_days(-1)):
         # print(i)
         run(i)
-
+    logging.info('广告数据清洗,结束')
 
 if __name__ == '__main__':
     # run('2021-05-18')