1

app/etl/data_stat_run.py

@@ -6,6 +6,7 @@ from sync_to_ck_task import *
 from app.etl.dm.dm_pitcher_daily_overview import dm_pitcher_daily_overview
 from app.etl.dw.dw_channel_daily import dw_daily_channel
 from app.etl.dw.dw_pitcher_daily import dw_pitcher_daily,dw_pitcher_trend
+from app.etl.ods.ods_book_info_daily import book_info_daily
 log=logger()
 du = DateUtils()
 
@@ -19,6 +20,7 @@ def do_cost(st,et):
     for i in du.getDateLists(st,et):
         channel_by_account_daily(i)
         channel_info_daily(i)
+        book_info_daily(i)
         dw_daily_channel_cost(i)
         dw_order_channel_cost_sync_ck(i)
 
@@ -28,8 +30,9 @@ def main(st,et):
     do_cost(st,et)
     dw_daily_channel()
     dw_pitcher_daily()
-    dm_pitcher_daily_overview()
     dw_pitcher_trend()
+    dm_pitcher_daily_overview()
+
 
 
 
@@ -48,8 +51,4 @@ def daily():
 
 
 if __name__ == '__main__':
-    do_cost('2021-01-20','2021-01-28')
-    dw_daily_channel()
-    dw_pitcher_daily()
-    dm_pitcher_daily_overview()
-    dw_pitcher_trend()
+   do_cost('2020-12-01','2021-01-28')

app/etl/data_stat_task.py

@@ -12,12 +12,12 @@ from sync_to_ck_task import dw_order_channel_cost_sync_ck
 
 def dw_daily_channel_cost(ymd):
     sql="""replace into dw_daily_channel_cost
-        select dt,x.channel,pitcher,stage,platform,book,
+        select x.dt,x.channel,pitcher,stage,x.platform,x.book,
                ifnull(view_count,0),ifnull(click_count,0),ifnull(follow_user,0),ifnull(cost,0)/100 as cost,
                ifnull(web_view_count,0) web_view_count,
                ifnull(platform_view_count,0) platform_view_count,
                ifnull(web_order_count,0) web_order_count,
-               ifnull(type,'') 
+               ifnull(type,''),ifnull(require_roi,0),ifnull(require_mult,0)
                  from
         (select dt,channel,stage,pitcher,platform,book from channel_info_daily where dt='{0}' and channel!='') x  
         left join
@@ -39,7 +39,10 @@ def dw_daily_channel_cost(ymd):
         (select account_id,channel from channel_by_account_daily where dt='{0}') b on a.account_id=b.account_id  group by channel) 
         z on x.channel=z.channel 
         left join (SELECT channel,type from channel_by_account_daily GROUP By channel,type) k on x.channel=k.channel
-        
+        left join (
+        select dt,book,platform,require_roi,require_mult from ods_book_info_daily 
+        )m on x.book=m.book and x.platform=m.platform and x.dt=m.dt
+
         """.format(ymd)
     print(sql)
 
@@ -167,7 +170,12 @@ def   dw_channel_daily_total(ymd):
 
 
 if __name__ == '__main__':
+    for i in dt.getDateLists('2019-03-18','2021-01-27'):
+        channel_by_account_daily(i)
+        channel_info_daily(i)
+
     # dw_daily_pitcher('2021-01-14')
+    # dw_daily_channel_cost('2021-01-28')
 
     exit(0)
     # dw_channel_daily_total('2020-07-20')

app/etl/dw/dw_channel_daily.py

@@ -19,9 +19,9 @@ select
        view_count,click_count,follow_user,cost,reg_order_count,reg_order_user,
        reg_order_amount,reg_order_amount30,web_view_count,platform_view_count,web_order_count,total_cost,
        total_amount,reg_order_user_again,reg_order_user7,reg_order_user30,reg_order_amount7,type,
-       total_first_amount 
+       total_first_amount,require_roi,require_mult
 from
-(select dt,channel, pitcher,stage,platform,book,cost,view_count,click_count,follow_user,web_view_count,platform_view_count,web_order_count,type from dw_daily_channel_cost) a
+(select dt,channel, pitcher,stage,platform,book,cost,view_count,click_count,follow_user,web_view_count,platform_view_count,web_order_count,type,require_roi,require_mult from dw_daily_channel_cost) a
     left outer join
     (select toDate(formatDateTime(reg_time,'%Y-%m-%d')) as dt2,channel as channel2,
        sum(amount) as reg_order_amount,
@@ -63,7 +63,7 @@ select formatDateTime(reg_time,'%Y-%m-%d') reg_date,channel,count(1) user_order_
     print("ok")
 
     data=ck.execute('select * from dw_daily_channel')
-    insert_sql="insert into dw_channel_daily values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
+    insert_sql="insert into dw_channel_daily values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
     st=time.time()
     db.quchen_text.execute("truncate table dw_channel_daily")
     db.quchen_text.executeMany(insert_sql,data)

app/etl/dw/dw_pitcher_daily.py

@@ -16,40 +16,21 @@ def dw_pitcher_daily():
     from dw_daily_channel group by dt,pitcher
     """
     data = ck.execute(sql)
-    db.quchen_text.execute('truncate table dw_pitcher_daily')
     db.quchen_text.executeMany('replace into dw_pitcher_daily values (%s,%s,%s,%s,%s,%s,%s,%s)',data)
 
-    add_total()
-
-
-def add_total():
-    sql='select pitcher from dw_pitcher_daily group by pitcher'
-    pitchers= [i[0] for i in db.quchen_text.getData(sql)]
-
-    sql2="""replace into dw_pitcher_daily
-    SELECT x.dt,x.pitcher,amount,cost,reg_amount,first_reg_amount,x.total_cost,x.total_amount FROM
-    
-    (SELECT dt,pitcher,@total_sum:=@total_sum+reg_amount total_amount,@total_cost:=@total_cost+cost total_cost from 
-        (SELECT dt,pitcher,reg_amount,cost FROM dw_pitcher_daily where pitcher='{}' order by dt) b,(SELECT @total_sum:=0,@total_cost:=0) a 
-    ) x 
-    left join dw_pitcher_daily y on x.dt=y.dt and x.pitcher=y.pitcher"""
-
-    for i in pitchers:
-        db.quchen_text.execute(sql2.format(i))
-
-    db.quchen_text.execute('delete from dw_pitcher_daily where total_cost=0')
-
-
 def dw_pitcher_trend():
     sql="""select a.dt,pitcher,
        sum(cost),
        sum(order_amount),
        sum(reg_order_amount),
+       sum(first_order_amount),
+       sum(first_order_amount*require_mult-cost) inva_cost,
        sum(r1) d1,sum(r2) d2,sum(r3) d3,sum(r4) d4,sum(r5) d5,sum(r6) d6,sum(r7) d7,sum(r8) d8,sum(r9) d9,
        sum(r10) d10,sum(r11) d11,sum(r12) d12,
        sum(r13) d13,sum(r14) d14,sum(r15) d15,sum(r16) d16,sum(r17) d17,sum(r18) d18,sum(r19) d19,
        sum(r20) d20,sum(r21) d21,sum(r22) d22,sum(r23) d23,sum(r24) d24,sum(r25) d25,
        sum(r26) d26,sum(r27) d27,sum(r28) d28,sum(r29) d29,sum(r30) d30
+       
 from dw_daily_channel a
 left outer join
 (select toDate(formatDateTime(reg_time,'%Y-%m-%d')) dt,channel,
@@ -88,8 +69,8 @@ group by pitcher,a.dt"""
 
     data=ck.execute(sql)
     db.quchen_text.execute("truncate table dw_pitcher_trend")
-    db.quchen_text.executeMany("replace into dw_pitcher_trend values (%s,%s,%s,%s,%s,%s,"
-                           "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,"
+    db.quchen_text.executeMany("replace into dw_pitcher_trend values (%s,%s,%s,%s,%s,%s,%s,"
+                           "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,"
                            "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",data)
 
 

app/etl/ods/ods_book_info_daily.py

@@ -0,0 +1,31 @@
+
+import time
+from model.DateUtils import DateUtils
+from model.DataBaseUtils import MysqlUtils,CkUtils
+du = DateUtils()
+db= MysqlUtils()
+ck = CkUtils()
+
+
+def book_info_daily(ymd):
+
+    sql=f"""SELECT '{ymd}' dt,b.book_name book,c.platform_name  platform,node_price require_roi,ratio require_mult from
+(
+select y.platform_book_id,node_price,ratio FROM
+(select max(create_time) create_time,platform_book_id from t_platform_book_ratio where effect_day<='{ymd}' and del_flag=0 GROUP BY platform_book_id ) x
+left join t_platform_book_ratio y on x.platform_book_id=x.platform_book_id 
+and x.create_time=y.create_time
+) a
+left join t_platform_book b on a.platform_book_id=b.id 
+left join t_platform_novel c on b.platform_id=c.id 
+    """
+    data =db.zx.getData(sql)
+    print(data)
+    db.quchen_text.executeMany("replace into ods_book_info_daily values(%s,%s,%s,%s,%s)",data)
+
+
+if __name__ == '__main__':
+    # get_book_info('2021-01-28')
+    book_info_daily()
+
+

config/db_config.yaml

@@ -2,4 +2,10 @@ quchen_text:
   host: rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com
   user: superc
   passwd: Cc719199895
-  db: quchen_text
+  db: quchen_text
+
+zx:
+  host: rm-bp145mi6r24ik50z5xo.mysql.rds.aliyuncs.com
+  user: readonly
+  passwd: toDataOfReadOnly$2020
+  db: zx_prod

model/DataBaseUtils.py

@@ -14,6 +14,7 @@ log = logger()
 
 class MysqlUtils:
     _quchen_text = None
+    _zx=None
 
     def __init__(self):
         p_path =os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
@@ -35,6 +36,15 @@ class MysqlUtils:
                                       db=conf['db'])
         return self._quchen_text
 
+    @property
+    def zx(self):
+        conf = self.config['zx']
+        self._zx = MysqlOperation(host=conf['host'],
+                                  user=conf['user'],
+                                  passwd=conf['passwd'],
+                                  db=conf['db'])
+        return self._zx
+
 
 
     def find_db(self, db):
@@ -51,6 +61,11 @@ class MysqlUtils:
             self._quchen_text.cursor.close()
             self._quchen_text.conn.close()
 
+        if self._zx:
+            self._zx.cursor.close()
+            self._zx.conn.close()
+
+
 
 class CkUtils: