New Updates

Author: Joshua Mayanja

.env

@@ -0,0 +1,3 @@
+ASTRA_DB_CLIENT_ID=oorvSWmoNPLetRKkQytSrMzm
+ASTRA_DB_CLIENT_SECRET=SR9g6xHuSmI5S+qPsNWiODwO1JiWZaaCCFU2pkP+MGzjt4HZ-0ujoo2to5dAB01sMfM_PCTg9OMqz8-fsEHgLYa.-XfUW-mLZiHlxkPuPF+-PIw+HpWyqIgt3YU2p-gq
+ASTRA_DB_CLIENT_TOKEN=AstraCS:oorvSWmoNPLetRKkQytSrMzm:3a16792c6faa9df4b2e38a2733ecc98cdef97bbab3892c9fccc7dae591332701

.gitignore

@@ -1,5 +1,6 @@
 env/
 spam-classifier/
 zips/
+ignored/
 *.pkl
 

app/__pycache__/config.cpython-37.pyc

@@ -0,0 +1,16 @@
+from functools import lru_cache
+import os 
+
+from pydantic import BaseSettings, Field
+
+os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = '1'
+class Settings(BaseSettings):
+    db_client_id: str = Field(..., env="ASTRA_DB_CLIENT_ID")
+    db_client_secret: str = Field(..., env="ASTRA_DB_CLIENT_SECRET")
+
+    class Config:
+        env_file = '.env'
+
+@lru_cache(maxsize=None)
+def get_settings():
+    return Settings()

app/__pycache__/db.cpython-37.pyc

@@ -0,0 +1,30 @@
+import pathlib
+from cassandra.cluster import Cluster
+from cassandra.auth import PlainTextAuthProvider
+from cassandra.cqlengine import connection
+
+from app.config import get_settings
+
+
+BASE_DIR = pathlib.Path(__file__).resolve().parent
+CLUSTER_BUNDLE = str(BASE_DIR / 'ignored' / 'astradb_connect.zip')
+
+settings = get_settings()
+
+ASTRA_DB_CLIENT_ID = settings.db_client_id
+ASTRA_DB_CLIENT_SECRET = settings.db_client_secret
+
+def get_cluster():
+    cloud_config= {
+        'secure_connect_bundle': CLUSTER_BUNDLE
+    }
+    auth_provider = PlainTextAuthProvider(ASTRA_DB_CLIENT_ID, ASTRA_DB_CLIENT_SECRET)
+    cluster = Cluster(cloud=cloud_config, auth_provider=auth_provider)
+    return cluster
+
+def get_session():
+    cluster = get_cluster()
+    session = cluster.connect()
+    connection.register_connection(str(session), session=session)
+    connection.set_default_connection(str(session))
+    return session

app/__pycache__/encoders.cpython-37.pyc

@@ -0,0 +1,20 @@
+import json
+import numpy as np
+
+class NumpyEncoder(json.JSONEncoder):
+    """Special JSONEncoder for Numpy types."""
+    def default(self, obj):
+        if isinstance(obj, np.integer):
+            return int(obj)
+        elif isinstance(obj, np.floating):
+            return float(obj)
+        elif isinstance(obj, np.ndarray):
+            return obj.tolist()
+        return json.JSONEncoder.default(self, obj)
+
+
+def encode_to_json(data, as_py=True):
+    encoded = json.dumps(data, cls=NumpyEncoder)
+    if as_py:
+        encoded = json.loads(encoded)
+    return encoded

app/__pycache__/main.cpython-37.pyc

@@ -1,18 +1,26 @@
-import json
-from multiprocessing.spawn import spawn_main
 import pathlib
 from typing import Optional
 from fastapi import FastAPI
-from keras.models import load_model
-from keras_preprocessing.text import tokenizer_from_json
-from keras_preprocessing.sequence import pad_sequences
+from fastapi.responses import StreamingResponse
+from app.config import get_settings
+from app.db import get_session
+
+from app.ml import SpamModel
+from app.models import SpamInference
+
+from cassandra.cqlengine.management import sync_table
+from cassandra.query import SimpleStatement
+
+from app.schema import Query
 
 app = FastAPI(
     version="1.0.0",
     title="DrexSpam",
     description="An Artificial Intelligence based Spam detector API using machine learning",
 )
 
+SETTINGS = get_settings()
+
 BASE_DIR = pathlib.Path(__file__).resolve().parent
 
 MODEL_DIR = BASE_DIR.parent / "models"
@@ -22,34 +30,60 @@
 SPAM_METADATA_PATH = SPAM_MODEL_DIR / "spam-classifer-metadata.json"
 
 SPAM_MODEL = None
-SPAM_TOKENIZER = None
-SPAM_METADATA = {}
-LEGEND_INVERTED = {}
+DB_SESSION = None
+SPAM_INFERENCE = SpamInference
 
 @app.on_event("startup")
 def on_startup():
-    global SPAM_MODEL, SPAM_TOKENIZER, SPAM_METADATA, LEGEND_INVERTED
-    # Load model
-    if SPAM_MODEL_PATH.exists():
-        SPAM_MODEL = load_model(SPAM_MODEL_PATH)
-    if SPAM_TOKENIZER_PATH.exists():
-        t_json = SPAM_TOKENIZER_PATH.read_text()
-        SPAM_TOKENIZER = tokenizer_from_json(t_json)
-    if SPAM_METADATA_PATH.exists():
-        SPAM_METADATA = json.loads(SPAM_METADATA_PATH.read_text())
-        LEGEND_INVERTED = SPAM_METADATA["labels_legend_inverted"]
-
-def predict(query: str):
-    sequences = SPAM_TOKENIZER.texts_to_sequences([query])
-    maxlen = SPAM_METADATA.get("max_sequence") or 280
-    x_input = pad_sequences(sequences, maxlen=280)
-    preds_array = SPAM_MODEL.predict(x_input)
-    return {}
-
+    global SPAM_MODEL, DB_SESSION
+    SPAM_MODEL = SpamModel(
+        model_path = SPAM_MODEL_PATH,
+        tokenizer_path = SPAM_TOKENIZER_PATH,
+        metadata_path= SPAM_METADATA_PATH,
+    )
+    DB_SESSION = get_session()
+    sync_table(SPAM_INFERENCE)
 
 @app.get("/")
 def read_index(q: Optional[str] = None):
-    global SPAM_MODEL, SPAM_METADATA
-    query = q or "Hello world"
-    print(SPAM_MODEL)
-    return {"query": query, **SPAM_METADATA}
+    return {"hello": "world"}
+
+@app.post("/")
+def create_infercence(q: Query):
+    global SPAM_MODEL
+    query = q.query or "Hello world"
+    preds_dict = SPAM_MODEL.predict_text(query)
+    top = preds_dict.get("top")
+    data = {"query": query, **top}
+    obj = SPAM_INFERENCE.objects.create(**data)
+    return obj
+
+@app.get("/inferences")
+def get_inferences():
+    q = SPAM_INFERENCE.objects.all()
+    return list(q)
+
+@app.get("/inferences/{my_uuid}")
+def get_inference_detail(my_uuid):
+    obj = SPAM_INFERENCE.objects.get(uuid=my_uuid)
+    return obj 
+
+def fetch_row(statement: SimpleStatement, fetch_size: int, session=None):
+    statement.fetch_size = fetch_size
+    result_set = session.execute(statement)
+    has_pages = result_set.has_more_pages
+    yield "uuid,label,confidence,query,model_version\n"
+    while has_pages:
+        for row in result_set.current_rows:
+            yield f"{row['uuid']},{row['label']},{row['confidence']},{row['query']},{row['model_version']}\n"
+        has_pages = result_set.has_more_pages
+        result_set = session.execute(statement, paging_state=result_set.paging_state)
+
+@app.get("/dataset")
+def export_inferences():
+    global DB_SESSION
+    cql_query = "SELECT * FROM spam_inferences.spam_inference LIMIT 10000"
+    # rows = DB_SESSION.execute(cql_query)
+    statement = SimpleStatement(cql_query)
+    return StreamingResponse(fetch_row(statement, 25, DB_SESSION))
+

app/__pycache__/ml.cpython-37.pyc

@@ -0,0 +1,102 @@
+from dataclasses import dataclass
+import json
+import numpy as np
+from pathlib import Path
+from typing import Any, List, Optional
+from importlib_metadata import metadata
+from keras.models import load_model
+from keras_preprocessing.sequence import pad_sequences
+from keras_preprocessing.text import tokenizer_from_json
+
+from app.encoders import NumpyEncoder, encode_to_json
+
+
+@dataclass
+class SpamModel:
+    """Drex Machine Learning Spam Classifier Model"""
+
+    model_path: Path
+    metadata_path: Optional[Path] = None
+    tokenizer_path: Optional[Path] = None
+
+    model = None
+    tokenizer = None
+    metadata = None
+
+    def __post_init__(self):
+        if self.model_path.exists():
+            self.model = load_model(self.model_path)
+        if self.tokenizer_path:
+            if self.tokenizer_path.exists():
+                if self.is_json(self.tokenizer_path):
+                    tokenizer_text = self.tokenizer_path.read_text()
+                    self.tokenizer = tokenizer_from_json(tokenizer_text)
+        if self.metadata_path:
+            if self.metadata_path.exists():
+                if self.is_json(self.metadata_path):
+                    metadata_text = self.metadata_path.read_text()
+                    self.metadata = json.loads(metadata_text)
+
+    def get_model(self):
+        if not self.model:
+            raise Exception("Model not loaded")
+        return self.model
+
+    def get_tokenizer(self):
+        if not self.tokenizer:
+            raise Exception("Tokenizer not loaded")
+        return self.tokenizer
+
+    def get_metadata(self):
+        if not self.metadata:
+            raise Exception("Metadata not loaded")
+        return self.metadata
+
+    def get_sequences_from_text(self, texts: List[str]):
+        tokenizer = self.get_tokenizer()
+        sequences = tokenizer.texts_to_sequences(texts)
+        return sequences
+
+    def get_input_from_sequences(self, sequences: List[Any]):
+        metadata = self.get_metadata()
+        maxlen = metadata.get("max_sequence") or 280
+        x_input = pad_sequences(sequences, maxlen)
+        return x_input
+
+    def get_label_legend_inverted(self):
+        metadata = self.get_metadata()
+        legend = metadata.get("labels_legend_inverted") or {}
+        if len(legend.keys()) != 2:
+            raise Exception("Legend invalid")
+        return legend
+
+    def get_label_pred(self, index: int, val):
+        label_legend_inverted = self.get_label_legend_inverted()
+        labeled_pred = {
+            "label": label_legend_inverted[str(index)],
+            "confidence": val,
+        }
+        return labeled_pred
+
+    def get_top_label_pred(self, preds):
+        top_index = np.argmax(preds)
+        top_pred = self.get_label_pred(top_index, preds[top_index])
+        return top_pred
+
+    def is_json(self, path: Path):
+        if path.name.endswith(".json"):
+            return True
+        return False
+
+    def predict_text(self, query: str, include_top=True, encode_json=True):
+        model = self.get_model()
+        sequences = self.get_sequences_from_text([query])
+        x_input = self.get_input_from_sequences(sequences)
+        preds_array = model.predict(x_input)[0]
+        preds = [self.get_label_pred(i, x) for i, x in enumerate(list(preds_array))]
+        results = {"predictions": preds}
+        if include_top:
+            results["top"] = self.get_top_label_pred(preds_array)
+        if encode_json:
+            results = encode_to_json(results)
+        return results

app/__pycache__/models.cpython-37.pyc

@@ -0,0 +1,11 @@
+import uuid
+from cassandra.cqlengine import columns
+from cassandra.cqlengine.models import Model
+
+class SpamInference(Model):
+    __keyspace__ = "spam_inferences"
+    uuid = columns.UUID(primary_key=True, default=uuid.uuid1)
+    query = columns.Text()
+    label = columns.Text()
+    confidence = columns.Float()
+    model_version = columns.Text(default="v1")

app/__pycache__/schema.cpython-37.pyc

@@ -0,0 +1,4 @@
+from pydantic import BaseModel
+
+class Query(BaseModel):
+    query: str

app/config.py

@@ -7,6 +7,6 @@
         "ham": 0,
         "spam": 1
     },
-    "max_sequence": 300,
+    "max_sequence": 280,
     "max_words": 280
 }