fix

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/deployment.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PublishConfigData" autoUpload="Always" serverName="root@8.154.46.64:22" remoteFilesAllowedToDisappearOnAutoupload="false">
+    <serverData>
+      <paths name="root@8.154.46.64:22">
+        <serverdata>
+          <mappings>
+            <mapping deploy="/tmp/pycharm_project_374" local="$PROJECT_DIR$" />
+          </mappings>
+        </serverdata>
+      </paths>
+    </serverData>
+    <option name="myAutoUpload" value="ALWAYS" />
+  </component>
+</project>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,47 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="34">
+            <item index="0" class="java.lang.String" itemvalue="motor" />
+            <item index="1" class="java.lang.String" itemvalue="scikit-learn" />
+            <item index="2" class="java.lang.String" itemvalue="sklearn" />
+            <item index="3" class="java.lang.String" itemvalue="baidu-aip" />
+            <item index="4" class="java.lang.String" itemvalue="elasticsearch5" />
+            <item index="5" class="java.lang.String" itemvalue="kombu" />
+            <item index="6" class="java.lang.String" itemvalue="numpy" />
+            <item index="7" class="java.lang.String" itemvalue="requests" />
+            <item index="8" class="java.lang.String" itemvalue="redis" />
+            <item index="9" class="java.lang.String" itemvalue="celery" />
+            <item index="10" class="java.lang.String" itemvalue="elasticsearch" />
+            <item index="11" class="java.lang.String" itemvalue="thrift" />
+            <item index="12" class="java.lang.String" itemvalue="image_match" />
+            <item index="13" class="java.lang.String" itemvalue="pypinyin" />
+            <item index="14" class="java.lang.String" itemvalue="msgpack-python" />
+            <item index="15" class="java.lang.String" itemvalue="bson" />
+            <item index="16" class="java.lang.String" itemvalue="retry" />
+            <item index="17" class="java.lang.String" itemvalue="tornado" />
+            <item index="18" class="java.lang.String" itemvalue="pymongo" />
+            <item index="19" class="java.lang.String" itemvalue="zbarlight" />
+            <item index="20" class="java.lang.String" itemvalue="opencv-python" />
+            <item index="21" class="java.lang.String" itemvalue="snappy" />
+            <item index="22" class="java.lang.String" itemvalue="pydub" />
+            <item index="23" class="java.lang.String" itemvalue="scribe-logger" />
+            <item index="24" class="java.lang.String" itemvalue="python-redis-lock" />
+            <item index="25" class="java.lang.String" itemvalue="scribe" />
+            <item index="26" class="java.lang.String" itemvalue="oss2" />
+            <item index="27" class="java.lang.String" itemvalue="pykafka" />
+            <item index="28" class="java.lang.String" itemvalue="python-consul" />
+            <item index="29" class="java.lang.String" itemvalue="Pillow" />
+            <item index="30" class="java.lang.String" itemvalue="ujson" />
+            <item index="31" class="java.lang.String" itemvalue="openpyxl" />
+            <item index="32" class="java.lang.String" itemvalue="phantomjs" />
+            <item index="33" class="java.lang.String" itemvalue="mongodb" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (script_tools)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/scripts.iml" filepath="$PROJECT_DIR$/.idea/scripts.iml" />
+    </modules>
+  </component>
+</project>

.idea/remote-mappings.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="RemoteMappingsManager">
+    <list>
+      <list>
+        <remote-mappings server-id="python@sftp://root@8.154.46.64:22/usr/bin/python3" />
+      </list>
+    </list>
+  </component>
+</project>

.idea/scripts.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.8 (script_tools)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

cronjob/media/vd_compress.py

@@ -1,6 +1,5 @@
-import os
+from aliyunsdkcore.client import AcsClient
-import time
+from aliyunsdkcore.request import CommonRequest
-
 from lib.all import *
 import ffmpy
 import cv2
@@ -8,19 +7,34 @@ import oss2
 import subprocess
 
 service_name = 'vd_compress'
-# log_dir = '/app/log'
+log_dir = '/app/log'
-log_dir = '/Users/erwin/data/log'
+# log_dir = '/Users/erwin/data/log'
 logger = Logger(service_name, log_dir=log_dir, print_terminal=False)
 
 access_key_id = "LTAI5tAdu5LRvZwm4LJa21Fo"
 access_key_secret = "WGvSQsDralTfFAAxhEqLBOgbXqflHo"
-endpoint_internal = "https://oss-cn-hangzhou.aliyuncs.com"
+endpoint_internal = "https://oss-cn-hangzhou-internal.aliyuncs.com"
 bucket_name = "wishpal-ironfan-media"
 
 bucket = oss2.Bucket(oss2.Auth(access_key_id, access_key_secret), endpoint_internal, bucket_name)
 url_get_videos_by_status = "https://api.tiefen.fun/op/media/get_videos_by_status"
 url_update_video_compress = "https://api.tiefen.fun/op/media/update_video_compress"
-hw_cdn_host = "https://filecdn01.tiefen.fun/"
+cdn_host = "https://filecdn01.tiefen.fun/"
+acs_client = AcsClient(access_key_id, access_key_secret, 'cn-hangzhou')
+
+
+def refresh_cdn_cache(url):
+    request = CommonRequest()
+    request.set_domain('cdn.aliyuncs.com')
+    request.set_version('2018-05-10')
+    request.set_action_name('RefreshObjectCaches')
+    request.set_method('POST')
+
+    request.add_query_param('ObjectPath', url)
+    request.add_query_param('ObjectType', 'File')  # 可以是 'File' 或 'Directory'
+
+    response = acs_client.do_action_with_exception(request)
+    return response
 
 
 class VdHelper:
@@ -132,7 +146,7 @@ class S:
 
     def get_one_wait_compress_video(self):
         param = {
-            "ids": [130337],  # 52062
+            "ids": [],  # 52062
             "status": 0,
             "offset": 0,
             "limit": 1,
@@ -198,7 +212,7 @@ class S:
         elif content_type == "video/x-m4v":
             local_src_path_new = local_src_path + ".m4v"
         else:
-            logger.Error("invalid content_type, id: {}, src_id: {}, content_type: {}".format(video_id, hw_cdn_host + src_id, content_type))
+            logger.Error("invalid content_type, id: {}, src_id: {}, content_type: {}".format(video_id, cdn_host + src_id, content_type))
             return False
         os.renames(local_src_path, local_src_path_new)
 
@@ -210,20 +224,17 @@ class S:
         logger.Info("pixel_format: {}".format(pf))
 
         output_file = local_h264_720_path + ".mp4"
-        # self.to_del_files.append(output_file)
+        self.to_del_files.append(output_file)
         if pf not in ["yuv420p", "yuvj420p"]:
             # 先转成8bit, 再转成264 720p mp4
             output_file_8bit = local_h264_720_path + "_8bit_tmp.mp4"
             vdh.transfer_h264_720p_25fps_10bit_to_8bit(output_file_8bit, output_file)
+            self.to_del_files.append(output_file_8bit)
         else:
             # 转成264 720p mp4
             vdh.transfer_h264_720p_25fps(output_file)
 
         # 上传
-        # 如果文件存在，则先删除再上传
-        if bucket.object_exists(src_id_h264_720):
-            ret_oss = bucket.delete_object(src_id_h264_720)
-            logger.Info("{} already exists in oss, delete status: {}".format(src_id_h264_720, ret_oss.status))
         upload_ret = self.upload_video_to_oss(output_file, src_id_h264_720)
         upload_ret_status = upload_ret.status
         logger.Info("upload_ret, {}, {}".format(src_id, upload_ret_status))
@@ -233,12 +244,16 @@ class S:
         # 更新db
         db_ret = self.set_compress_finish(video_id, file_size, "", 0, src_id_h264_720, output_file_size, 100, int(time.time()))
 
+        # 刷新cdn
+        file_url = cdn_host + src_id_h264_720
+        refresh_res = refresh_cdn_cache(file_url)
+        logger.Info("refresh ali cdn, url: {}, res: {}".format(file_url, refresh_res))
         # vdh.remove()
         # os.remove(output_file)
 
         logger.Info("before, {}, {}, {}".format(src_id, content_type, file_size))
         logger.Info("after_h264720, {}, {}, {}, {}".format(src_id_h264_720, output_content_type, output_file_size, db_ret))
-        logger.Info("host: {}".format(hw_cdn_host + src_id_h264_720))
+        logger.Info("host: {}".format(file_url))
         return True
 
 
@@ -317,4 +332,3 @@ while True:
             s.set_compress_finish(video_id, 0, "", 0, "", 0, -200, int(time.time()))
         else:
             s.set_compress_finish(video_id, 0, "", 0, "", 0, -100, int(time.time()))
-    break

media/calc_10bit.py

@@ -52,7 +52,7 @@ class S:
             "src_id": 1,
             "src_id_720": 1
         }
-        docs = self.col_video.find({}, projection=prj)
+        docs = self.col_video.find({"pixel_format": {"$exists": False}}, projection=prj)
 
         total = len(docs)
         idx = 0