wishpal_ironfan
/
scripts
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix

This commit is contained in:
lwl0608 2024-09-14 16:16:15 +08:00
parent 1d152d07f0
commit c2f1761ea9
11 changed files with 143 additions and 17 deletions

8
.idea/.gitignore vendored Normal file
View File

@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

15
.idea/deployment.xml Normal file
View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="PublishConfigData" autoUpload="Always" serverName="root@8.154.46.64:22" remoteFilesAllowedToDisappearOnAutoupload="false">
<serverData>
<paths name="root@8.154.46.64:22">
<serverdata>
<mappings>
<mapping deploy="/tmp/pycharm_project_374" local="$PROJECT_DIR$" />
</mappings>
</serverdata>
</paths>
</serverData>
<option name="myAutoUpload" value="ALWAYS" />
</component>
</project>

47
.idea/inspectionProfiles/Project_Default.xml Normal file
View File

@ -0,0 +1,47 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="34">
<item index="0" class="java.lang.String" itemvalue="motor" />
<item index="1" class="java.lang.String" itemvalue="scikit-learn" />
<item index="2" class="java.lang.String" itemvalue="sklearn" />
<item index="3" class="java.lang.String" itemvalue="baidu-aip" />
<item index="4" class="java.lang.String" itemvalue="elasticsearch5" />
<item index="5" class="java.lang.String" itemvalue="kombu" />
<item index="6" class="java.lang.String" itemvalue="numpy" />
<item index="7" class="java.lang.String" itemvalue="requests" />
<item index="8" class="java.lang.String" itemvalue="redis" />
<item index="9" class="java.lang.String" itemvalue="celery" />
<item index="10" class="java.lang.String" itemvalue="elasticsearch" />
<item index="11" class="java.lang.String" itemvalue="thrift" />
<item index="12" class="java.lang.String" itemvalue="image_match" />
<item index="13" class="java.lang.String" itemvalue="pypinyin" />
<item index="14" class="java.lang.String" itemvalue="msgpack-python" />
<item index="15" class="java.lang.String" itemvalue="bson" />
<item index="16" class="java.lang.String" itemvalue="retry" />
<item index="17" class="java.lang.String" itemvalue="tornado" />
<item index="18" class="java.lang.String" itemvalue="pymongo" />
<item index="19" class="java.lang.String" itemvalue="zbarlight" />
<item index="20" class="java.lang.String" itemvalue="opencv-python" />
<item index="21" class="java.lang.String" itemvalue="snappy" />
<item index="22" class="java.lang.String" itemvalue="pydub" />
<item index="23" class="java.lang.String" itemvalue="scribe-logger" />
<item index="24" class="java.lang.String" itemvalue="python-redis-lock" />
<item index="25" class="java.lang.String" itemvalue="scribe" />
<item index="26" class="java.lang.String" itemvalue="oss2" />
<item index="27" class="java.lang.String" itemvalue="pykafka" />
<item index="28" class="java.lang.String" itemvalue="python-consul" />
<item index="29" class="java.lang.String" itemvalue="Pillow" />
<item index="30" class="java.lang.String" itemvalue="ujson" />
<item index="31" class="java.lang.String" itemvalue="openpyxl" />
<item index="32" class="java.lang.String" itemvalue="phantomjs" />
<item index="33" class="java.lang.String" itemvalue="mongodb" />
</list>
</value>
</option>
</inspection_tool>
</profile>
</component>

6
.idea/inspectionProfiles/profiles_settings.xml Normal file
View File

@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

4
.idea/misc.xml Normal file
View File

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (script_tools)" project-jdk-type="Python SDK" />
</project>

8
.idea/modules.xml Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/scripts.iml" filepath="$PROJECT_DIR$/.idea/scripts.iml" />
</modules>
</component>
</project>

10
.idea/remote-mappings.xml Normal file
View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="RemoteMappingsManager">
<list>
<list>
<remote-mappings server-id="python@sftp://root@8.154.46.64:22/usr/bin/python3" />
</list>
</list>
</component>
</project>

8
.idea/scripts.iml Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.8 (script_tools)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

6
.idea/vcs.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

46
cronjob/media/vd_compress.py
View File

@ -1,6 +1,5 @@
import os
import time
from aliyunsdkcore.client import AcsClient
from aliyunsdkcore.request import CommonRequest
from lib.all import *
import ffmpy
import cv2
@ -8,19 +7,34 @@ import oss2
import subprocess
service_name = 'vd_compress'
# log_dir = '/app/log'
log_dir = '/Users/erwin/data/log'
log_dir = '/app/log'
# log_dir = '/Users/erwin/data/log'
logger = Logger(service_name, log_dir=log_dir, print_terminal=False)
access_key_id = "LTAI5tAdu5LRvZwm4LJa21Fo"
access_key_secret = "WGvSQsDralTfFAAxhEqLBOgbXqflHo"
endpoint_internal = "https://oss-cn-hangzhou.aliyuncs.com"
endpoint_internal = "https://oss-cn-hangzhou-internal.aliyuncs.com"
bucket_name = "wishpal-ironfan-media"
bucket = oss2.Bucket(oss2.Auth(access_key_id, access_key_secret), endpoint_internal, bucket_name)
url_get_videos_by_status = "https://api.tiefen.fun/op/media/get_videos_by_status"
url_update_video_compress = "https://api.tiefen.fun/op/media/update_video_compress"
hw_cdn_host = "https://filecdn01.tiefen.fun/"
cdn_host = "https://filecdn01.tiefen.fun/"
acs_client = AcsClient(access_key_id, access_key_secret, 'cn-hangzhou')
def refresh_cdn_cache(url):
request = CommonRequest()
request.set_domain('cdn.aliyuncs.com')
request.set_version('2018-05-10')
request.set_action_name('RefreshObjectCaches')
request.set_method('POST')
request.add_query_param('ObjectPath', url)
request.add_query_param('ObjectType', 'File') # 可以是 'File' 或 'Directory'
response = acs_client.do_action_with_exception(request)
return response
class VdHelper:
@ -132,7 +146,7 @@ class S:
def get_one_wait_compress_video(self):
param = {
"ids": [130337], # 52062
"ids": [], # 52062
"status": 0,
"offset": 0,
"limit": 1,
@ -198,7 +212,7 @@ class S:
elif content_type == "video/x-m4v":
local_src_path_new = local_src_path + ".m4v"
else:
logger.Error("invalid content_type, id: {}, src_id: {}, content_type: {}".format(video_id, hw_cdn_host + src_id, content_type))
logger.Error("invalid content_type, id: {}, src_id: {}, content_type: {}".format(video_id, cdn_host + src_id, content_type))
return False
os.renames(local_src_path, local_src_path_new)
@ -210,20 +224,17 @@ class S:
logger.Info("pixel_format: {}".format(pf))
output_file = local_h264_720_path + ".mp4"
# self.to_del_files.append(output_file)
self.to_del_files.append(output_file)
if pf not in ["yuv420p", "yuvj420p"]:
# 先转成8bit, 再转成264 720p mp4
output_file_8bit = local_h264_720_path + "_8bit_tmp.mp4"
vdh.transfer_h264_720p_25fps_10bit_to_8bit(output_file_8bit, output_file)
self.to_del_files.append(output_file_8bit)
else:
# 转成264 720p mp4
vdh.transfer_h264_720p_25fps(output_file)
# 上传
# 如果文件存在，则先删除再上传
if bucket.object_exists(src_id_h264_720):
ret_oss = bucket.delete_object(src_id_h264_720)
logger.Info("{} already exists in oss, delete status: {}".format(src_id_h264_720, ret_oss.status))
upload_ret = self.upload_video_to_oss(output_file, src_id_h264_720)
upload_ret_status = upload_ret.status
logger.Info("upload_ret, {}, {}".format(src_id, upload_ret_status))
@ -233,12 +244,16 @@ class S:
# 更新db
db_ret = self.set_compress_finish(video_id, file_size, "", 0, src_id_h264_720, output_file_size, 100, int(time.time()))
# 刷新cdn
file_url = cdn_host + src_id_h264_720
refresh_res = refresh_cdn_cache(file_url)
logger.Info("refresh ali cdn, url: {}, res: {}".format(file_url, refresh_res))
# vdh.remove()
# os.remove(output_file)
logger.Info("before, {}, {}, {}".format(src_id, content_type, file_size))
logger.Info("after_h264720, {}, {}, {}, {}".format(src_id_h264_720, output_content_type, output_file_size, db_ret))
logger.Info("host: {}".format(hw_cdn_host + src_id_h264_720))
logger.Info("host: {}".format(file_url))
return True
@ -317,4 +332,3 @@ while True:
s.set_compress_finish(video_id, 0, "", 0, "", 0, -200, int(time.time()))
else:
s.set_compress_finish(video_id, 0, "", 0, "", 0, -100, int(time.time()))
break

2
media/calc_10bit.py
View File

@ -52,7 +52,7 @@ class S:
"src_id": 1,
"src_id_720": 1
}
docs = self.col_video.find({}, projection=prj)
docs = self.col_video.find({"pixel_format": {"$exists": False}}, projection=prj)
total = len(docs)
idx = 0