Merge pull request #154 from yimm0815/smartupload

add ffmpeg transcode

Author: xwu2git

Diff for: deployment/docker-swarm/smart-upload.m4

@@ -1,10 +1,9 @@
-
-define(`SERVICE_INTERVAL_SMART_UPLOAD',`60')dnl
+define(`SERVICE_INTERVAL_SMART_UPLOAD',`120')dnl
 
     defn(`OFFICE_NAME')_smart_upload:
         image: smtc_smart_upload:latest
         environment:
-            QUERY: "time>=now-eval(defn(`SERVICE_INTERVAL_SMART_UPLOAD')*1000000) where objects.detection.bounding_box.x_max-objects.detection.bounding_box.x_min>0.01"
+            QUERY: "time>=now-eval(defn(`SERVICE_INTERVAL_SMART_UPLOAD')*1000) where objects.detection.bounding_box.x_max-objects.detection.bounding_box.x_min>0.01"
             INDEXES: "recordings,analytics"
             OFFICE: "defn(`OFFICE_LOCATION')"
             DBHOST: "http://ifelse(eval(defn(`NOFFICES')>1),1,defn(`OFFICE_NAME')_db,db):9200"
@@ -28,4 +27,9 @@ ifelse(defn(`PLATFORM'),`VCAC-A',`dnl
             placement:
                 constraints:
                     - defn(`OFFICE_ZONE')
+            resources:
+                limits:
+                    cpus: '0.20'
+                reservations:
+                    cpus: '0.10'
 

Diff for: deployment/kubernetes/smart-upload.yaml.m4

@@ -1,5 +1,5 @@
 include(office.m4)
-define(`SERVICE_INTERVAL_SMART_UPLOAD',60)dnl
+define(`SERVICE_INTERVAL_SMART_UPLOAD',120)dnl
 
 apiVersion: apps/v1
 kind: Deployment
@@ -21,6 +21,11 @@ spec:
         - name: defn(`OFFICE_NAME')-smart-upload
           image: smtc_smart_upload:latest
           imagePullPolicy: IfNotPresent
+          resources:
+            requests:
+                cpu: "100m"
+            limits:
+                cpu: "200m"
           env:
             - name: QUERY
               value: "time>=eval(defn(`SERVICE_INTERVAL_SMART_UPLOAD')*1000) where objects.detection.bounding_box.x_max-objects.detection.bounding_box.x_min>0.01"

Diff for: maintenance/smart-upload/Dockerfile

@@ -1,7 +1,7 @@
 
-FROM centos:7.6.1810
+FROM openvisualcloud/xeon-centos76-media-ffmpeg:latest
 
-RUN yum -y install epel-release && yum -y install python36-requests python36-ply && rm -rf /var/cache/yum/* curl
+RUN yum -y install epel-release && yum -y install python36-requests python36-ply python36-psutil && rm -rf /var/cache/yum/* curl
 
 COPY --from=smtc_common /*.py /home/
 COPY    *.py /home/

Diff for: maintenance/smart-upload/smart-upload.py

@@ -2,6 +2,7 @@
 
 from db_ingest import DBIngest
 from db_query import DBQuery
+from probe import probe, run
 from signal import signal, SIGTERM
 import requests
 import os
@@ -18,10 +19,24 @@
 smhost=os.environ["SMHOST"]
 cloudhost=os.environ["CLOUDHOST"]
 
+dbs=None
+rs=None
+
 def quit_service(signum, sigframe):
     if dbs and rs: dbs.delete(rs["_id"])
     exit(143)
 
+def upload(cloudhost, filename, office, sensor, timestamp):
+    with open(filename,"rb") as fd:
+        r=requests.post(cloudhost,data={
+            "time":timestamp,
+            "office":str(office[0])+","+str(office[1]),
+            "sensor":sensor,
+        },files={
+            "file": fd,
+        },verify=False)
+    os.remove(filename)
+
 signal(SIGTERM, quit_service)
 dbs=DBIngest(index="services",office=office,host=dbhost)
 while True:
@@ -39,40 +54,28 @@ def quit_service(signum, sigframe):
 dbq=DBQuery(index=indexes,office=office,host=dbhost)
 
 while True:
-    print("Sleeping...")
-    time.sleep(service_interval)
 
     print("Searching...",flush=True)
     print("query = ", query)
 
     try:
         for q in dbq.search(query):
-            filename=smhost+'/'+q["_source"]["path"]
-            print("filename: ", filename)
+            url=smhost+'/'+q["_source"]["path"]
+            print("url: ", url)
+
+            mp4file="/tmp/"+str(os.path.basename(url))
 
-            r=requests.head(filename, timeout=10)
-            if r.status_code!=200: 
-                print("pipeline status: "+str(r.status_code), flush=True)
-                print(r.text, flush=True)
-                break
+            print("Transcoding...")
+            os.remove(mp4file)
+            list(run(["/usr/bin/ffmpeg","-f","mp4","-i",url,"-c:v","libsvt_hevc","-c:a","aac",mp4file]))
 
-            #add updates to request
-            print("get url: ", r.url)
-            print("send to cloud storage: ", cloudhost)
-            print("office: ", str(office[0])+","+str(office[1]))
-            #print("sensor: ", sensor)
-            #print("year: ", year)
-            #print("month: ", month)
-            #print("day: ", day)
-            #print("time: ", str((os.path.basename(filename).split('.')[0]))
-            # r=requests.post(cloudhost,data={
-                # "office":str(office[0])+","+str(office[1]),
-                # "sensor":***,
-                # "year":***, 
-                # "month":***,
-                # "day":***,
-                # "time":str((os.path.basename(filename).split('.')[0])),
-                # })
+            print("Uploading: ", cloudhost)
+            sensor=q["_source"]["sensor"]
+            timestamp=q["_source"]["time"]
+            upload(cloudhost, mp4file, office, sensor, timestamp)
 
     except Exception as e:
-        print("Exception: "+str(e), flush=True)
+        print("Exception: "+str(e), flush=True)
+
+    print("Sleeping...")
+    time.sleep(service_interval)