FIX:分钟任务检查

app/crontab_task/minuteRun.py

@@ -5,7 +5,6 @@ from model.sql_models import DB
 from data_processing import video_processing
 from config import using_config as db_config
 
-
 def image_check():
     # TODO:图片素材，进行对应修改
     pass

app/etl/MaterialLibrary/MaterialDataClean.py

@@ -45,7 +45,6 @@ def description():
 
     df = db.dm.pd_data_sql(sql)
 
-
     df["data_type"] = 'all'
     df['type'] = '2'
     df['create_by'] = '0'
@@ -58,6 +57,8 @@ def description():
 
 
 def image():
+    # TODO:视频,图片新添加的列没有归并到media,idea列里面,在dw_image_cost_day也没有进行对应归并
+
     sql = """select signature,sum(consume_amount) consume_amount,
             sum(click_times) click_times,
             sum(view_times) view_times,
@@ -87,12 +88,12 @@ def image():
             replace (min(preview_url),' ,','') as content,
             if(is_video=1,2,1) type,  
             if(locate(',',signature)>0,0,1) single_img,
-            min(width) width ,
-			min(height) height,
+            if(min(width)>0,min(width),0) width,
+            if(min(height)>0,min(height),0) height,
 			min(replace(if(left (size ,2)='0,',substring(size ,3),size) ,',0','')) media_size,
             min(replace(format ,' ,','')) media_format,
-            min(video_length) video_length,
-            min(video_bit_rate) video_bit_rate
+            if(min(video_length)>0,min(video_length),0) video_length,
+            if(min(video_bit_rate)>0,min(video_bit_rate),0) video_bit_rate
             from dw_image_cost_day  
             where signature is not null and signature !=''  
             and length (replace (replace (signature,',',''),' ',''))>0
@@ -159,12 +160,12 @@ sum(cost) consume_amount,
 group_concat(distinct book) novels,
 min(dt) start_date,max(dt) end_date,
 min(replace(preview_url ,' ,','')) media,
-min(width) width ,
-min(height) height,
+if(min(width)>0,min(width),0) width,
+if(min(height)>0,min(height),0) height,
 min(replace(if(left (size ,2)='0,',substring(size ,3),size) ,',0','')) media_size ,
 min(replace(format ,' ,','')) media_format,
-min(video_length) video_length,
-min(video_bit_rate) video_bit_rate,
+if(min(video_length)>0,min(video_length),0) video_length,
+if(min(video_bit_rate)>0,min(video_bit_rate),0) video_bit_rate,
 type channel,
 if(is_video=1,2,1) type,
 if(locate(',',signature)>0,0,1) single_img
@@ -195,6 +196,9 @@ group by signature ,title ,article
     table = "t_ads_idea"
 
     db.zx_ads.dfsave2mysql(df, table, key, tag)
+    # TODO：线上，线下视频进行归并
+
+    # 同一个signature选择同一个
 
 
 def run():
@@ -210,6 +214,3 @@ if __name__ == '__main__':
     # description()
     # image()
     # adcreative()
-
-
-

app/etl/dw/dw_image_cost_day.py

@@ -109,7 +109,12 @@ def run(dt):
     # 进行数据存储
     db.dm.execute(f'delete from dw_image_cost_day where dt="{dt}"')
     db.dm.executeMany(
-        "replace into dw_image_cost_day values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
+        '''replace into dw_image_cost_day 
+        (dt,type,use_times,cost,view_count,click_count,follow_count,order_count,
+        order_amount,title,description,book,platform,stage,channel,pitcher,image_id,
+        preview_url,signature,is_video,width,height,size,format,video_length,
+        video_bit_rate,video_meta_data,download_path)
+        values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)''',
         data)
 
 

model/sql_models.py

@@ -1,7 +1,7 @@
 from sqlalchemy import create_engine, MetaData
 from sqlalchemy.orm import sessionmaker, scoped_session
 from config import using_config
-
+import urllib
 
 class DB():
     def __init__(self, config=None):
@@ -13,8 +13,8 @@ class DB():
     def DBengine(self):
         # 初始化数据库连接:
         db_uri = 'mysql+pymysql://' \
-                 '{username}:{password}@{host}:{port}/{database}'.format(username=self.config['username'],
-                                                                         password=self.config['password'],
+                 '{username}:{password}@{host}:{port}/{database}'.format(username=urllib.parse.quote(self.config['username']),
+                                                                         password=urllib.parse.quote(self.config['password']),
                                                                          host=self.config['host'],
                                                                          port=self.config['port'],
                                                                          database=self.config['database'])