MOD:ck partition bug ,fix

app/etl/dw/dw_image_cost_day.py

@@ -1,10 +1,9 @@
-import logging
+from logging import handlers
 from model.DataBaseUtils import MysqlUtils, CkUtils
 from model.DateUtils import DateUtils
 from model.DingTalkUtils import DingTalkUtils
-import pandas as pd
 from datetime import datetime
-
+import logging
 # logging.getLogger().setLevel(logging.WARNING)
 
 
@@ -155,8 +154,10 @@ def run(dt):
         _[23] = str(_[23]) if _[23] is not None else None
 
     col_str = ','.join(col)
+    logging.info('ck填充数据进入')
+    ck.client.execute('SET max_partitions_per_insert_block=1000;')
     ck.client.execute('insert into dw_image_cost_day ({}) values'.format(col_str), data)
-
+    logging.info('ck填充数据,结束')
 
 def hourly():
     try:
@@ -164,7 +165,8 @@ def hourly():
         run(du.getNow())
         logging.info('广告数据清洗,结束')
     except Exception as e:
-        DingTalkUtils().send("广告数据清洗失败\n"+str(e))
+        logging.error(str(e))
+        DingTalkUtils().send("广告数据清洗失败\n" + str(e))
 
 
 def day():
@@ -177,7 +179,20 @@ def day():
 
 if __name__ == '__main__':
     # run('2021-05-18')
-
-    for i in du.getDateLists(du.get_n_days(-495), du.get_n_days(0)):
+    #-495
+    logging.basicConfig(
+        handlers=[
+            logging.handlers.RotatingFileHandler('.ad_hourly.log',
+                                                 maxBytes=10 * 1024 * 1024,
+                                                 backupCount=5,
+                                                 encoding='utf-8')
+            , logging.StreamHandler()  # 供输出使用
+        ],
+        level=logging.INFO,
+        format="%(asctime)s - %(levelname)s %(filename)s %(funcName)s %(lineno)s - %(message)s"
+    )
+    for i in du.getDateLists(du.get_n_days(-370), du.get_n_days(0)):
         print(i)
         run(i)
+    # print(du.getNow())
+    # run(du.getNow())