DataCenterGame/app/etl/MaterialLibrary/MaterialDataClean.py

"""
desc : 素材库数据清洗
每日运行一次

"""
from model.DataBaseUtils import MysqlUtils
db =MysqlUtils()

def title():
    sql = """select REPLACE(REPLACE(title, CHAR(10), ''), CHAR(13), '') content,
        sum(cost) consume_amount,
        sum(click_count) click_times,
        sum(view_count) view_times,
        group_concat(distinct book) novels,
        max(dt) end_date,min(dt) start_date
        from dw_image_cost_day where title!='' and title is not null  GROUP BY REPLACE(REPLACE(title, CHAR(10), ''), CHAR(13), '')  limit 1000
				"""

    df =  db.dm.getData_pd(sql)
    print(df)
    df["data_type"] = 'all'
    df['type'] = 1
    df['create_by'] = 0

    key = ["content", "type"]
    tag = ["view_times", "click_times", "novels", "start_date", "end_date", "create_by",'data_type', 'consume_amount']
    table = "t_ads_content"

    db.zx_test.dfsave2mysql(df, table, key, tag)

def description():
    sql = """select REPLACE(REPLACE(description, CHAR(10), ''), CHAR(13), '') content,
            sum(cost) consume_amount,
            sum(click_count) click_times,
            sum(view_count) view_times,
            group_concat(distinct book) novels,
            max(dt) end_date,min(dt) start_date
            from dw_image_cost_day where description!='' and description is not null  GROUP BY REPLACE(REPLACE(description, CHAR(10), ''), CHAR(13), '')  limit 1000
    				"""

    df = db.dm.getData_pd(sql)
    print(df)

    df["data_type"] = 'all'
    df['type'] = 2
    df['create_by'] = 0
    key = ["content", "type"]
    tag = ["view_times", "click_times", "novels", "start_date", "end_date", "create_by",'data_type', 'consume_amount']
    table = "t_ads_content"

    db.zx_test.dfsave2mysql(df, table, key, tag)


def image():
    sql="""select signature,
            sum(cost) consume_amount,
            sum(click_count) click_times,
            sum(view_count) view_times,
            group_concat(distinct book) novels ,
            max(dt) end_date,
            min(dt) start_date,
            min(preview_url) content,
            if(flag=1,2,1) type,  
            if(locate(',',signature)>0,0,1) single_img
            from dw_image_cost_day  where signature is not null and signature !=''  GROUP BY  signature,flag limit 1000"""

    df = db.dm.getData_pd(sql)
    # print(df)

    df['create_by'] = 0
    df["data_type"] = 'all'

    key = ["signature"]
    tag = ["view_times", "click_times", "novels", "start_date", "end_date", "create_by", "single_img", "content",'consume_amount','type']
    table = "t_ads_media"

    db.zx_test.dfsave2mysql(df, table, key, tag)


def adcreative():
    sql="""select signature,title,description article,
sum(click_count) click_times,
sum(view_count) view_times,
sum(cost) consume_amount,
group_concat(distinct book) novels,
max(dt) start_date,min(dt) end_date,
min(preview_url) media,
type channel,
if(flag=1,2,1) type,
if(locate(',',signature)>0,0,1) single_img

from dw_image_cost_day where signature is not null and signature!=''   GROUP BY  signature,title,description,type,flag
				"""

    df = db.dm.getData_pd(sql)

    key = ["signature",'title','article']
    tag = ["view_times", "click_times", "novels", "start_date", "end_date","type","channel",'consume_amount','single_img','media']
    table = "t_ads_idea"

    db.zx_test.dfsave2mysql(df, table, key, tag)


if __name__ == '__main__':
    # title()
    # description()
    # image()
    adcreative()