新增定时任务

Pipfile

@@ -10,6 +10,7 @@ pyyaml = "*"
 numpy = "==1.19.3"
 requests = "*"
 clickhouse-driver = "==0.1.4"
+selenium = "*"
 
 [dev-packages]
 

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "38999f2b06e220db7a7ef6d36319633c2e26d52979cb42b33f3cb686b0b785b5"
+            "sha256": "3a5ca47fefc981ac811f0e1b6938d734e35eb770daf6b9dce6f4a625f83136ba"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -18,10 +18,10 @@
     "default": {
         "certifi": {
             "hashes": [
-                "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd",
-                "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4"
+                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
+                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
             ],
-            "version": "==2020.11.8"
+            "version": "==2020.12.5"
         },
         "chardet": {
             "hashes": [
@@ -124,33 +124,33 @@
         },
         "pandas": {
             "hashes": [
-                "sha256:09e0503758ad61afe81c9069505f8cb8c1e36ea8cc1e6826a95823ef5b327daf",
-                "sha256:0a11a6290ef3667575cbd4785a1b62d658c25a2fd70a5adedba32e156a8f1773",
-                "sha256:0d9a38a59242a2f6298fff45d09768b78b6eb0c52af5919ea9e45965d7ba56d9",
-                "sha256:112c5ba0f9ea0f60b2cc38c25f87ca1d5ca10f71efbee8e0f1bee9cf584ed5d5",
-                "sha256:185cf8c8f38b169dbf7001e1a88c511f653fbb9dfa3e048f5e19c38049e991dc",
-                "sha256:3aa8e10768c730cc1b610aca688f588831fa70b65a26cb549fbb9f35049a05e0",
-                "sha256:41746d520f2b50409dffdba29a15c42caa7babae15616bcf80800d8cfcae3d3e",
-                "sha256:43cea38cbcadb900829858884f49745eb1f42f92609d368cabcc674b03e90efc",
-                "sha256:5378f58172bd63d8c16dd5d008d7dcdd55bf803fcdbe7da2dcb65dbbf322f05b",
-                "sha256:54404abb1cd3f89d01f1fb5350607815326790efb4789be60508f458cdd5ccbf",
-                "sha256:5dac3aeaac5feb1016e94bde851eb2012d1733a222b8afa788202b836c97dad5",
-                "sha256:5fdb2a61e477ce58d3f1fdf2470ee142d9f0dde4969032edaf0b8f1a9dafeaa2",
-                "sha256:6613c7815ee0b20222178ad32ec144061cb07e6a746970c9160af1ebe3ad43b4",
-                "sha256:6d2b5b58e7df46b2c010ec78d7fb9ab20abf1d306d0614d3432e7478993fbdb0",
-                "sha256:8a5d7e57b9df2c0a9a202840b2881bb1f7a648eba12dd2d919ac07a33a36a97f",
-                "sha256:8b4c2055ebd6e497e5ecc06efa5b8aa76f59d15233356eb10dad22a03b757805",
-                "sha256:a15653480e5b92ee376f8458197a58cca89a6e95d12cccb4c2d933df5cecc63f",
-                "sha256:a7d2547b601ecc9a53fd41561de49a43d2231728ad65c7713d6b616cd02ddbed",
-                "sha256:a979d0404b135c63954dea79e6246c45dd45371a88631cdbb4877d844e6de3b6",
-                "sha256:b1f8111635700de7ac350b639e7e452b06fc541a328cf6193cf8fc638804bab8",
-                "sha256:c5a3597880a7a29a31ebd39b73b2c824316ae63a05c3c8a5ce2aea3fc68afe35",
-                "sha256:c681e8fcc47a767bf868341d8f0d76923733cbdcabd6ec3a3560695c69f14a1e",
-                "sha256:cf135a08f306ebbcfea6da8bf775217613917be23e5074c69215b91e180caab4",
-                "sha256:e2b8557fe6d0a18db4d61c028c6af61bfed44ef90e419ed6fadbdc079eba141e"
+                "sha256:0a643bae4283a37732ddfcecab3f62dd082996021b980f580903f4e8e01b3c5b",
+                "sha256:0de3ddb414d30798cbf56e642d82cac30a80223ad6fe484d66c0ce01a84d6f2f",
+                "sha256:19a2148a1d02791352e9fa637899a78e371a3516ac6da5c4edc718f60cbae648",
+                "sha256:21b5a2b033380adbdd36b3116faaf9a4663e375325831dac1b519a44f9e439bb",
+                "sha256:24c7f8d4aee71bfa6401faeba367dd654f696a77151a8a28bc2013f7ced4af98",
+                "sha256:26fa92d3ac743a149a31b21d6f4337b0594b6302ea5575b37af9ca9611e8981a",
+                "sha256:2860a97cbb25444ffc0088b457da0a79dc79f9c601238a3e0644312fcc14bf11",
+                "sha256:2b1c6cd28a0dfda75c7b5957363333f01d370936e4c6276b7b8e696dd500582a",
+                "sha256:2c2f7c670ea4e60318e4b7e474d56447cf0c7d83b3c2a5405a0dbb2600b9c48e",
+                "sha256:3be7a7a0ca71a2640e81d9276f526bca63505850add10206d0da2e8a0a325dae",
+                "sha256:4c62e94d5d49db116bef1bd5c2486723a292d79409fc9abd51adf9e05329101d",
+                "sha256:5008374ebb990dad9ed48b0f5d0038124c73748f5384cc8c46904dace27082d9",
+                "sha256:5447ea7af4005b0daf695a316a423b96374c9c73ffbd4533209c5ddc369e644b",
+                "sha256:573fba5b05bf2c69271a32e52399c8de599e4a15ab7cec47d3b9c904125ab788",
+                "sha256:5a780260afc88268a9d3ac3511d8f494fdcf637eece62fb9eb656a63d53eb7ca",
+                "sha256:70865f96bb38fec46f7ebd66d4b5cfd0aa6b842073f298d621385ae3898d28b5",
+                "sha256:731568be71fba1e13cae212c362f3d2ca8932e83cb1b85e3f1b4dd77d019254a",
+                "sha256:b61080750d19a0122469ab59b087380721d6b72a4e7d962e4d7e63e0c4504814",
+                "sha256:bf23a3b54d128b50f4f9d4675b3c1857a688cc6731a32f931837d72effb2698d",
+                "sha256:c16d59c15d946111d2716856dd5479221c9e4f2f5c7bc2d617f39d870031e086",
+                "sha256:c61c043aafb69329d0f961b19faa30b1dab709dd34c9388143fc55680059e55a",
+                "sha256:c94ff2780a1fd89f190390130d6d36173ca59fcfb3fe0ff596f9a56518191ccb",
+                "sha256:edda9bacc3843dfbeebaf7a701763e68e741b08fccb889c003b0a52f0ee95782",
+                "sha256:f10fc41ee3c75a474d3bdf68d396f10782d013d7f67db99c0efbfd0acb99701b"
             ],
             "index": "pypi",
-            "version": "==1.1.4"
+            "version": "==1.1.5"
         },
         "pymysql": {
             "hashes": [
@@ -201,6 +201,14 @@
             "index": "pypi",
             "version": "==2.25.0"
         },
+        "selenium": {
+            "hashes": [
+                "sha256:2d7131d7bc5a5b99a2d9b04aaf2612c411b03b8ca1b1ee8d3de5845a9be2cb3c",
+                "sha256:deaf32b60ad91a4611b98d8002757f29e6f2c2d5fcaf202e1c9ad06d6772300d"
+            ],
+            "index": "pypi",
+            "version": "==3.141.0"
+        },
         "six": {
             "hashes": [
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",

app/etl/data_stat_daily.py

@@ -0,0 +1,23 @@
+from model.DateUtils import DateUtils
+from data_stat_task import *
+
+du = DateUtils()
+
+yestoday = du.get_n_days(-1)
+today = du.get_n_days()
+
+def _channel_by_account_daily():
+    channel_by_account_daily(today)
+    channel_by_account_daily(du.get_n_days(yestoday))
+
+
+def _channel_info_daily():
+    channel_info_daily(today)
+    channel_info_daily(yestoday)
+
+if __name__ == '__main__':
+    _channel_by_account_daily()
+    _channel_info_daily()
+
+
+

app/etl/data_stat_task.py

@@ -51,6 +51,78 @@ def dw_daily_channel(dt):
     db.quchen_text.execute(insert_sql)
 
 
+
+def channel_by_account_daily(ymd):
+    """返回当天消耗账户对应的公众号表"""
+    sql="""replace into channel_by_account_daily 
+            select  '{0}' as dt,a.account_id as account_id, ifnull(ifnull(b.name,a.name),'') as channel  from
+            (select  account_id,name from advertiser_qq) a
+                left join
+            (select b.account_id,b.name from
+            (select min(end_time) as end_time,account_id from account_change where end_time>'{0}' GROUP BY account_id) a
+            left join account_change  b on  a.end_time=b.end_time and a.account_id=b.account_id) b on a.account_id=b.account_id""".format(ymd)
+    db.quchen_text.execute(sql)
+
+def channel_info_daily(ymd):
+    """获取公众号某天的期数，投手，平台，书籍
+    @ return [[]]
+    """
+    # 获取现在的全量公众号信息
+    sql="""select '{}' as dt,a.name ,ifnull(stage,''),ifnull(pitcher,''),ifnull(platform,''),ifnull(book,'') from (
+            select  name from advertiser_vx  where name is not null group by name--  公众号全量表
+            union
+            select  name from advertiser_qq where name is not null group by name
+            union
+            select  name from account_change group by name
+            union
+            select channel as name   from pitcher_change group by channel
+            union
+            select  name from platform_change group by name
+            union
+            select  name from book_change group by name) a left join (
+            select name,ifnull(stage,'') stage,ifnull(pitcher,'') pitcher,ifnull(platform,'') platform,ifnull(book,'') book from advertiser_qq where name is not null  group by name,stage,pitcher,platform,book
+             union
+                select name,ifnull(stage,'') stage,ifnull(pitcher,'') pitcher,ifnull(platform,'') platform,ifnull(book,'') book from advertiser_vx where name is not null
+                ) b on a.name=b.name""".format(ymd)
+    data=db.quchen_text.get_data_list(sql)
+
+
+
+    pitcher_change=db.quchen_text.getData(
+                                          "select b.channel as channel,pitcher from "
+                                          "(select min(end_time) as end_time,channel from pitcher_change "
+                                     " where end_time>'{}' GROUP BY channel) a"
+                                     " left join pitcher_change  b on  a.end_time=b.end_time and a.channel=b.channel".format(ymd))
+
+    platform_change=db.quchen_text.getData("select b.name as channel,primary_platform as platform from (select min(change_date) as change_date,name from platform_change "
+                                          "where change_date>'{}' GROUP BY name) a "
+                                          "left join platform_change  b on  a.change_date=b.change_date and a.name=b.name".format(ymd))
+
+    book_change=db.quchen_text.getData("select b.name as channel,book from (select min(end_time) as end_time,name from book_change "
+                                           "where end_time>'{}' GROUP BY name) a "
+                                           "left join book_change  b on  a.end_time=b.end_time and a.name=b.name".format(ymd))
+
+    for i in data:
+        for j in pitcher_change:
+            if i[1]==j[0]:
+                i[3]=j[1]
+        for k in platform_change:
+            if i[1]==k[0]:
+                i[4]=k[1]
+        for h in book_change:
+            if i[1]==h[0]:
+                i[5]=h[1]
+
+
+    print(data)
+    insert_sql="replace into channel_info_daily values (%s,%s,%s,%s,%s,%s) "
+
+    db.quchen_text.executeMany(insert_sql,data)
+
+
+
+
+
 def clean_order(dt):
     sql="""	replace into ods_order
 			select 
@@ -68,10 +140,19 @@ def clean_order(dt):
 			""".format(dt)
     db.quchen_text.execute(sql)
 
-if __name__ == '__main__':
 
-    li=dt.getDateLists('2019-03-01','2020-01-01')
-    for i in  li:
+def order_account_text():
+    db.quchen_text.execute("truncate order_account_text")
+    with open('./wending_account_config.csv',encoding='utf-8') as f:
+
+        for i in f.readlines():
+            db.quchen_text.execute("insert into order_account_text(platform,text) values ('文鼎','{}')".format(i))
+
+
+
+if __name__ == '__main__':
+    # channel_info_daily('2020-12-14')
+    for i in dt.getDateLists('2019-03-18','2020-12-16'):
         print(i)
-        clean_order(i)
+        channel_info_daily(i)