From c2edffb5ad7767ce18538a7bee9bf65633f0de0f Mon Sep 17 00:00:00 2001 From: lwl0608 Date: Sun, 12 May 2024 22:21:26 +0800 Subject: [PATCH] day detail --- bi/vas_day_detail.py | 140 ++++++++++++++++++++++++++++++++++++++++++ ff | 142 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 282 insertions(+) create mode 100644 bi/vas_day_detail.py create mode 100644 ff diff --git a/bi/vas_day_detail.py b/bi/vas_day_detail.py new file mode 100644 index 0000000..5d1e859 --- /dev/null +++ b/bi/vas_day_detail.py @@ -0,0 +1,140 @@ +from lib.all import * + +service_name = 'vas_day_detail' +log_dir = '/app/log' +logger = Logger(service_name, log_dir=log_dir) + +hds = [ + "时间", + "会员数", "会员流水", + "空间解锁数", "空间解锁流水", + "空间超粉数", "空间超粉流水", + "空间动态数", "空间动态流水", + "其他数(微信+金币)", "其他流水(微信+金币)", + "总流水", "主播分成", "净收入", + "利润率", + "会员流水占比", + "空间解锁占比", + "空间超粉占比", + "空间动态占比", + "其他占比", +] + + +class S: + def __init__(self, st, et): + self.st = st + self.et = et + self.mysql_db_vas = Mysql( + "rm-bp11t1616a1kjvmx5.mysql.rds.aliyuncs.com", 3306, "vas", "root", "Wishpal2024" + ) + self.mysql_db_bi = Mysql( + "127.0.0.1", 3306, "metabase_bi", "metabase_bi", "Wishpal_2024" + ) + + def __del__(self): + self.mysql_db_vas.close() + self.mysql_db_bi.close() + + def get_product_sold_list(self): + sql = ''' +select product_id, + count(1) cnt, + sum(pay_amount) money +from vas_order +where ct>={} and ct<{} +and order_status in (1,2) +group by product_id +'''.format(self.st, self.et) + docs = self.mysql_db_vas.query(sql) + return docs + + def get_streamer_dias(self): + sql = ''' +select sum(`change`) dias +from vas_ch_income +where ct>={} and ct<{} +and mid>0 +'''.format(self.st, self.et) + docs = self.mysql_db_vas.query(sql) + if len(docs) > 0: + return safe_get_int(docs[0], "dias") + return 0 + + def insert_day_detail(self, data: tuple): + sql = '''insert into vas_day_detail ( + ptime, membership_cnt, membership_flow, zone_admission_cnt, zone_admission_flow, + zone_superfan_cnt, zone_superfan_flow, zone_moment_cnt, zone_moment_flow, + other_cnt, other_flow, total_flow, streamer_flow, official_flow, + profit_rate, membership_rate, zone_admission_rate, zone_superfan_rate, zone_moment_rate, other_rate + ) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)''' + return self.mysql_db_bi.exec(sql, data) + + def proc(self): + total_money = 0 # 总流水 + membership_cnt = 0 + membership_money = 0 + zone_admission_cnt = 0 + zone_admission_money = 0 + zone_superfan_cnt = 0 + zone_superfan_money = 0 + zone_moment_cnt = 0 + zone_moment_money = 0 + other_cnt = 0 + other_money = 0 + + sold_list = self.get_product_sold_list() + for sold in sold_list: + product_id = safe_get_str(sold, "product_id") + cnt = safe_get_int(sold, "cnt") + money = int(safe_get_int(sold, "money") / 100) + total_money += money + if product_id == "membership": + membership_cnt += cnt + membership_money += money + elif product_id == "h5_zone_admission": + zone_admission_cnt += cnt + zone_admission_money += money + elif product_id == "h5_zone_superfanship": + zone_superfan_cnt += cnt + zone_superfan_money += money + elif product_id == "h5_zone_moment": + zone_moment_cnt += cnt + zone_moment_money += money + else: + other_cnt += cnt + other_money += money + streamer_dias = self.get_streamer_dias() + streamer_money = int(streamer_dias / 10) + official_money = total_money - streamer_money + data = ( + datetime.datetime.fromtimestamp(self.st).strftime('%Y-%m-%d %H:%M:%S'), + membership_cnt, membership_money, + zone_admission_cnt, zone_admission_money, + zone_superfan_cnt, zone_superfan_money, + zone_moment_cnt, zone_moment_money, + other_cnt, other_money, + total_money, streamer_money, official_money, + safe_div(official_money, total_money), + safe_div(membership_money, total_money), + safe_div(zone_admission_money, total_money), + safe_div(zone_superfan_money, total_money), + safe_div(zone_moment_money, total_money), + safe_div(other_money, total_money) + ) + self.insert_day_detail(data) + + +# st_et_str_map = gen_st_et_str_map_v2( +# "2024-04-15 00:00:00", "2024-05-06 00:00:00" +# ) + +st_et_str_map = gen_st_et_str_map( + "2024-04-15 00:00:00", 1 +) + +for st_str, et_str in st_et_str_map.items(): + s = S( + get_ts_by_str(st_str), get_ts_by_str(et_str) + ) + s.proc() diff --git a/ff b/ff new file mode 100644 index 0000000..13cf60c --- /dev/null +++ b/ff @@ -0,0 +1,142 @@ +# 安装consul +mkdir /data/consul +cd /data/consul +wget https://releases.hashicorp.com/consul/1.18.1/consul_1.18.1_linux_amd64.zip + +** 注意解压后只有一个consul执行文件 +unzip consul_1.18.1_linux_amd64.zip + + +# 启动consul +nohup /data/consul/consul agent -server -data-dir=/tmp/consul -bootstrap -ui -advertise=172.31.37.68 -client=172.31.37.68 > /dev/null 2>&1 & + + + +# 安装nginx +mkdir /data +mkdir /data/nginx +cd /data/nginx +wget http://nginx.org/download/nginx-1.26.0.tar.gz +tar -zxvf nginx-1.26.0.tar.gz + +yum -y install pcre-devel +yum install -y zlib-devel + +cd /data/nginx/nginx-1.26.0 +./configure --prefix=/data/nginx + +安装：make +第一次安装：make install + +日志：/data/nginx/logs + +# 通过nginx-upsync-module和nginx_upstream_check_module模块进行编译 +mkdir /data/nginx/modules +cd /data/nginx/modules +# 这里是Github的资源，不能用wget下载，具体是： +nginx-upsync-module需要下载release里面的最新版本：v2.1.3 【wget https://github.com/weibocom/nginx-upsync-module/archive/refs/tags/v2.1.3.tar.gz】 +nginx_upstream_check_module需要下载整个项目的源码，主要用到靠近当前版本的补丁，使用patch命令进行补丁升级 + + +cd /data/nginx/nginx-1.26.0 +patch -p1 < /data/nginx/modules/nginx_upstream_check_module-master/check_1.20.1+.patch +./configure --prefix=/data/nginx --add-module=/data/nginx/modules/nginx_upstream_check_module-master --add-module=/data/nginx/modules/nginx-upsync-module-2.1.3 +make +make install + + +# 下载upsync模块并且将其解压 +wget -c https://github.com/weibocom/nginx-upsync-module/archive/master.zip +unzip nginx-upsync-module-master.zip + +# 下载nginx +wget -c http://nginx.org/download/nginx-1.9.9.tar.gz + +# 解压到当前目录 +tar -zxvf nginx-1.9.9.tar.gz + +# 配置一个nginx的用户以及用户组，-s /sbin/nologin nginx代表该用户是无法登录到主机的 +groupadd nginx +useradd -g nginx -s /sbin/nologin nginx + +# 这两个文件夹会在编译nginx时指定 +mkdir -p /var/tmp/nginx/client/ +mkdir -p /usr/local/nginx + +# 进入到nginx的解压后文件夹的目录下 +cd nginx-1.9.9 + +# 编译 nginx ，--prefix 代表nginx安装的目录。其中指定了用户和用户组，以及上面创建的文件夹并且添加了upsync模块，由于upsync解压在nginx同级目录下，所以这里使用..来指定到它 +./configure --prefix=/data/nginx --with-http_ssl_module --with-http_flv_module --with-http_stub_status_module --with-http_gzip_static_module --with-http_realip_module --http-client-body-temp-path=/var/tmp/nginx/client/ --http-proxy-temp-path=/var/tmp/nginx/proxy/ --http-fastcgi-temp-path=/var/tmp/nginx/fcgi/ --http-uwsgi-temp-path=/var/tmp/nginx/uwsgi --http-scgi-temp-path=/var/tmp/nginx/scgi --with-pcre --add-module=/data/nginx/modules/nginx-upsync-module-2.1.3 +make && make install + +## 进入到刚刚nginx安装的目录，也就是/data/nginx目录中，进入conf目录中编辑conf目录的文件内容 + +upstream myserver { + server 127.0.0.1:11111;# 这个固定的，不用理 + # springbootserver ： key的值，upsync_timeout 超时时间3秒，upsync_interval 间隔时间 ，upsync_type 类型consul，strong_dependency 增强依赖 + upsync 192.168.254.134:8500/v1/kv/upstreams/springbootserver upsync_timeout=3000ms upsync_interval=500ms upsync_type=consul strong_dependency=off; + # 将拉取下来的配置文件放在以下配置的目录中 + upsync_dump_path /usr/local/nginx-1.9.9/conf/upsync_dump.conf; + + } + # 将server中的location指定为刚刚创建的upstream(上游服务器) + location / { + proxy_pass http://myserver; + index index.html index.htm; + } + +# /data/nginx/conf/nginx.conf部分片段 +http { + include mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + + upstream mixtest { + # 这里是consul的leader节点的HTTP端点 + upsync 172.31.37.68:8500/v1/kv/upstreams/mixtest upsync_timeout=6m upsync_interval=500ms upsync_type=consul strong_dependency=off; + # consul访问不了的时候的备用配置 + upsync_dump_path /data/nginx/app.conf; + # 这里是为了兼容Nginx的语法检查 + include /data/nginx/app.conf; + # 下面三个配置是健康检查的配置 + check interval=1000 rise=2 fall=2 timeout=3000 type=http default_down=false; + check_http_send "HEAD / HTTP/1.0\r\n\r\n"; + check_http_expect_alive http_2xx http_3xx; + } + + server { + listen 80; + server_name localhost; + location / { + proxy_pass http://app; + } + # 健康检查 - 查看负载均衡的列表 + location /upstream_list { + upstream_show; + } + # 健康检查 - 查看负载均衡的状态 + location /upstream_status { + check_status; + access_log off; + } + } +} + +# /data/nginx/app.conf +server 127.0.0.1:9000 weight=1 fail_timeout=10 max_fails=3; +server 127.0.0.1:9001 weight=1 fail_timeout=10 max_fails=3; + +# 进入到sbin目录下，启动nginx +./nginx + + +nohup ./app -p 10300 >/dev/null 2>&1 & +nohup ./app -p 10301 >/dev/null 2>&1 & + + +10301 + +curl -X PUT -d '{"weight":1, "max_fails":2, "fail_timeout":10}' http://172.31.37.68:8500/v1/kv/upstreams/app/127.0.0.1:10300 +curl -X PUT -d '{"weight":1, "max_fails":2, "fail_timeout":10}' http://172.31.37.68:8500/v1/kv/upstreams/app/127.0.0.1:10301