example implementation of the vision api

src/facebook_example/api/vision.clj

@@ -0,0 +1,59 @@
+(ns facebook-example.api.vision
+  (:gen-class)
+  (:require [clojure.string :as s]
+            [org.httpkit.client :as http]
+            [clojure.data.json :as json]
+            [environ.core :refer [env]])
+  (import (org.apache.commons.codec.binary Base64)))
+
+(def VISION-KEY (env :vision-key))
+
+;;; (download-image url) downloads an url as a byte array
+(defn download-image [url]
+  (let [response (http/get url {:as :byte-array})]
+    (:body @response)))
+
+;;; (image-to-b64 image) base64 encodes a image byte array
+(defn image-to-b64 [image]
+  (String. (Base64/encodeBase64 image)))
+
+;;; (request-body b64-image) generates a request body for the given b64 encoded image
+;;; for description of FACE_DETECTION & LABEL_DETECTION check out:
+;;; https://cloud.google.com/vision/docs/detecting-faces#vision-face-detection-protocol
+;;; https://cloud.google.com/vision/docs/detecting-labels#vision-label-detection-protocol
+;;; for other available features check out:
+;;; https://cloud.google.com/vision/docs/how-to
+;;; eg:
+;;; {"type" "IMAGE_PROPERTIES"}
+;;; adds :imagePropertiesAnnotation to response (see bot.clj:37)
+(defn request-body [b64-image]
+  {"requests" [{"image" {"content" b64-image}
+                "features" [{"type" "FACE_DETECTION"}
+                            {"type" "LABEL_DETECTION"}]}]})
+
+;;; (call-vision-api (b64-image) sends a generated request to the GC Vision API
+;;; see https://cloud.google.com/vision/docs/how-to for features
+;;; and corresponding responses
+(defn call-vision-api [b64-image]
+    (as-> b64-image data
+          @(http/post "https://vision.googleapis.com/v1/images:annotate"
+                     {  :query-params {"key" VISION-KEY}
+                        :headers {"Content-Type" "application/json"}
+                        :body (json/write-str (request-body data))
+                        :insecure? true})))
+
+;;; (handle-vision-response @response) handles the response from GC Vision API
+(defn handle-vision-response [{:keys [status headers body error]}]
+  (if (= status 200)
+      (-> (json/read-str body :key-fn keyword)
+          :responses
+          first)
+      (do (println "ERROR: Call to Vision API failed. Maybe check your vision-key in profiles.clj?")
+          (println body))))
+
+;;; (analyze url) takes an image url and returns a response from GC Vision API
+(defn analyze [url]
+  (-> (download-image url)
+      image-to-b64
+      call-vision-api
+      handle-vision-response))

src/facebook_example/incoming.clj

@@ -72,6 +72,24 @@
       (= postback "GET_LEMMINGS_BOTS") (outgoing/send-lemmings-bots)
       :else (outgoing/error))))
 
+(def compliments ["These are beautiful eyes!" "I like this nose!" "Wow, those lips!" "Nice ears!"])
+(defn on-image [sender-id attachment]
+  ;;; see vision.clj:22 and https://cloud.google.com/vision/docs/how-to
+  ;;; for further information about the vision api
+  (let [vision-response (vision/analyze (get-in attachment [:payload :url]))]
+    (cond
+      (contains? vision-response :faceAnnotations)
+      (let [face-annotations (:faceAnnotations vision-response)]
+        (fb/send-message sender-id (fb/text-message (rand-nth compliments))))
+      (contains? vision-response :labelAnnotations)
+      (let [label-annotations (:labelAnnotations vision-response)]
+        (let [firstLabel (:description (first label-annotations))]
+          (fb/send-message sender-id (fb/text-message (str "Is that your " firstLabel "? It's beautiful!")))))
+      :else (fb/send-message sender-id (fb/text-message "Uhm, I'm not sure what that is, but its beautiful!")))))
+
+(defn on-audio [sender-id attachment]
+  (fb/send-message sender-id (fb/text-message "That sounds beautiful!")))
+
 (defn on-attachments [event]
   ; Called by handle-message when the user has sent a file or sticker
   (println "on-attachment event:")
@@ -81,4 +99,8 @@
         time-of-message (get-in event [:timestamp])
         attachments (get-in event [:message :attachments])
         user-data (facebook/get-user-profile sender-id)]
-    (outgoing/thank-for-attachment)))
+    (let [attachment (first attachments)]
+      (cond
+        (= (:type attachment) "image") (on-image sender-id attachment)
+        (= (:type attachment) "audio") (on-audio sender-id attachment)
+        :else (outgoing/thank-for-attachment)))))