order 同步到ck

app/etl/rds_order_ck.py

@@ -0,0 +1,80 @@
+"""把rds的订单数据同步到clickhouse
+@auth ck
+@date 20201127
+"""
+
+from model.util.DataBaseUtils import MysqlUtils
+from model.util.ClickHouseUtils import CkUtils
+from model.util.DateUtils import DateUtils
+from datetime import datetime
+from model.common.log import logger
+log=logger()
+db = MysqlUtils()
+ck = CkUtils()
+dt = DateUtils()
+
+
+
+
+def run(mydt):
+    log.debug("run [{}] data".format(mydt))
+    col = "date,stage,platform,channel,channel_id,user_id,order_time,reg_time,amount,from_novel,order_id"
+
+    sql ="select * from `order` where date=UNIX_TIMESTAMP('{}') and platform!='阳光'".format(mydt)
+
+    data = db.quchen_text.getData(sql)
+
+    # 掌中云时间格式转换
+    data1 = []
+    for x in data:
+        li = list(x)
+
+        li[6] = li[6].replace('T',' ').replace('+08:00','').replace('/','-')
+
+        if li[6] == None or li[6] == '':
+            li[6] = '0000-00-00 00:00:00'
+
+        if li[7]==None or li[7]=='':
+            li[7]='0000-00-00 00:00:00'
+        else:
+            li[7] = li[7].replace('T',' ').replace('+08:00','').replace('/','-')
+
+        try:
+            li[6]=str(datetime.strptime(li[6],'%Y-%m-%d %H:%M'))
+            li[7]=str(datetime.strptime(li[7],'%Y-%m-%d %H:%M'))
+            # li[7]=str(datetime.strptime(li[7],'%Y/%m/%d %H:%M:%S'))
+        except:
+            pass
+            # print("======================")
+        #     try:
+        #         li[6] = str(datetime.strptime(li[6][:17], '%Y-%m-%d %H:%M'))
+        #         li[7] = str(datetime.strptime(li[7][:17], '%Y/%m/%d %H:%M'))
+        #     except:
+        #         pass
+                # li[6]='2020-11-01 00:00:00'
+                # li[7]='2020-11-01 00:00:00'
+
+        li[6] =li[6][:19]
+        li[7] =li[7][:19]
+
+        li[9] = '' if li[9] is None else li[9]
+        li[5] = '' if li[5] is None else li[5]
+        li[1] = '' if li[1] is None else li[1]
+
+        data1.append(tuple(li))
+
+    # 删除分区
+    ck.execute("alter table order drop  partition '{}' ".format(mydt))
+    ck.insertMany("order", col, tuple(data1))
+
+
+if __name__ == '__main__':
+    # dt_li =dt.getDateLists('2019-03-18','2020-01-01')
+    # for i in dt_li:
+    #     run(i)
+
+    today=dt.get_n_days()
+    run(today)
+
+
+

app/novel_daily_order_amount.py

@@ -1,38 +0,0 @@
-""""
-@desc 订单数据日统计
-@auth ck
-@date 20201119
-"""
-
-from model.util.DateUtils import DateUtils
-from model.util.DataBaseUtils import MysqlUtils
-db = MysqlUtils()
-dt = DateUtils()
-
-
-def run(ymd):
-
-    sql = """replace into novel_daily_order_amount
-             select "{0}" as ymd,platform,channel_id,channel,stage,
-                    ROUND(SUM(amount),2) as amount,
-                    count(1) as order_num,
-                    count(DISTINCT user_id) as order_user
-             from `order` 
-             where `date` = UNIX_TIMESTAMP("{0}") GROUP BY platform,stage,channel 
-             """.format(ymd)
-    db.quchen_text.execute(sql)
-
-
-if __name__ == '__main__':
-    ymd = dt.get_n_days(-1)
-    run(ymd)
-
-    # for i in dt.getDateLists("2020-10-01", "2020-11-01"):
-    #     run(i)
-
-
-
-
-
-
-