from model.DateUtils import DateUtils from app.api_data.tx_ad_cost import get_cost_older from app.etl.data_stat_run import do_cost from app.etl.dm.dm_pitcher_daily_overview import dm_pitcher_daily_overview from app.etl.dw.dw_channel_daily import dw_channel_daily from app.etl.dw.dw_pitcher_daily import dw_pitcher_trend from app.etl.src.src_book_info import src_book_info from app.etl.dw.dw_book_trend import book_trend from model.DateUtils import DateUtils from model.DataBaseUtils import MysqlUtils from app.api_data.tx_ad_cost.get_cost_older import get_v_data, mysql_insert_daily_vx, get_q_data, mysql_insert_daily_qq import logging import time from logging import handlers db = MysqlUtils() du = DateUtils() def get_data(st, et): # 1.获取数据 print(st, et) get_cost_older.run(st, et) # 同步到ck do_cost(st, et) # 2.数据处理 src_book_info() # 书籍卡点信息 # book_annual_expect_profit.run() # 年预期收益 dw_channel_daily() dw_pitcher_trend() book_trend() dm_pitcher_daily_overview() def get_data_vx(channel, st, et): # 用于处理单个微信号相关信息 sql = ''' select account_id,wechat_account_id,access_token,refresh_token,name, ifnull(stage,''),ifnull(pitcher,''),ifnull(platform,''),ifnull(book,'') from advertiser_vx where account_id in (select account_id from channel_by_account_daily where channel ='{}' and dt in (select max(dt) from channel_by_account_daily cbad) order by dt desc ) '''.format(channel) token_list_v = db.quchen_text.getData(sql) print(token_list_v) time1 = time.time() li = [] for y in token_list_v: get_v_data(y, li, st, et) for _ in li: print(_) print('get_daily_vx:' + str(len(li)) + 'cost:' + str(int(time.time() - time1))) mysql_insert_daily_vx(li) # do_cost(st, et) # src_book_info() # 书籍卡点信息 # dw_channel_daily() # dw_pitcher_trend() # book_trend() # dm_pitcher_daily_overview() def get_data_gdt(channel, st, et): # 用于处理单个微信号相关信息 sql = '''select account_id,'',access_token,refresh_token,name, ifnull(stage,''),ifnull(pitcher,''),ifnull(platform,''),ifnull(book,'') from advertiser_qq where account_id in (select account_id from channel_by_account_daily where channel ='{}' and dt in (select max(dt) from channel_by_account_daily cbad) order by dt desc ) '''.format(channel) print(sql) token_list_v = db.quchen_text.getData(sql) print(token_list_v) time1 = time.time() li = [] for y in token_list_v: get_q_data(y, li, st, et) for _ in li: print(_) print('get_daily_qq:' + str(len(li)) + 'cost:' + str(int(time.time() - time1))) mysql_insert_daily_qq(li) do_cost(st, et) src_book_info() # 书籍卡点信息 dw_channel_daily() dw_pitcher_trend() book_trend() dm_pitcher_daily_overview() if __name__ == "__main__": logging.basicConfig( handlers=[ logging.handlers.RotatingFileHandler('./cost_data.log', maxBytes=10 * 1024 * 1024, backupCount=5, encoding='utf-8') , logging.StreamHandler() # 供输出使用 ], level=logging.INFO, format="%(asctime)s - %(levelname)s %(filename)s %(funcName)s %(lineno)s - %(message)s" ) st = du.get_n_days(-50) et = du.get_n_days(-10) # get_data_vx(channel='落枫文海', st=st, et=et) # get_data_gdt(channel='落枫文海', st=st, et=et) # get_data(st,et) from app.etl import data_stat_task data_stat_task.dw_daily_channel_cost('2019-07-12')