Update integration with Google Gemini API; replace Cohere references, add jeannie.py for content generation, and modify README accordingly

Author: dan5e3s6ares

.gitignore

@@ -7,6 +7,7 @@ __pycache__/
 pynews*
 format.json
 export.py
+worker.py
 
 # C extensions
 *.so

README.md

@@ -30,15 +30,15 @@ Este Scraper utiliza IA para que não seja necessário a sua atualização const
     https://docs.docker.com/compose/install/
 
 
-### Crie uma chave junto a COHERE
+### Crie uma chave GEMINI API junto a Google
 
-- https://docs.cohere.com/
+- https://aistudio.google.com/
 <p>
 
-Atualize o valor em prompt.py
+Atualize o valor em jeannie.py
 ```
 ...
-os.environ["COHERE_API_KEY"] = "<YOUR-API-KEY>"
+GOOGLE_API_KEY = "<YOUR-API-KEY>"
 ...
 ```
 

app/bibliotecas.py

@@ -44,7 +44,7 @@
     "Django": {
         "library_name": "Django",
         "releases_url": "https://pypi.org/project/Django/",
-        "logo": "https://static.djangoproject.com/img/logos/django-logo-negative.png",
+        "logo": "https://www.djangoproject.com/m/img/logos/django-logo-positive.png",
         "repository": "https://github.com/django/django",
     },
     "Seaborn": {

app/getNews.py

@@ -1,7 +1,6 @@
 import asyncio
 import json
 import sys
-import time
 from datetime import datetime
 from json import dump
 
@@ -81,6 +80,9 @@ async def fetch(self, lib, url_name):
                     pynews[lib["library_name"]] = response
             elif response.get("news"):
                 pynews[lib["library_name"]] = response
+                pynews[lib["library_name"]]["library_name"] = lib[
+                    "library_name"
+                ]
                 pynews[lib["library_name"]]["logo"] = bibliotecas[
                     lib["library_name"]
                 ]["logo"]
@@ -100,15 +102,7 @@ async def main_loop(self, url_name, list_libs):
         await asyncio.gather(*tasks)
 
     async def get(self, url_name):
-        for i in range(0, len(self.study_case), 10):
-            batch = {
-                k: self.study_case[k]
-                for k in list(self.study_case.keys())[i : i + 10]
-            }
-            await self.main_loop(url_name, list(batch.keys()))
-            time.sleep(
-                60
-            )  # O Cohere gratuíto permite somente 10 requisições por minuto
+        await self.main_loop(url_name, list(self.study_case.keys()))
 
 
 async def main():

app/jeannie.py

@@ -0,0 +1,14 @@
+import google.generativeai as genai
+
+GOOGLE_API_KEY = "<YOUR-API-KEY>"
+
+genai.configure(api_key=GOOGLE_API_KEY)
+
+model = genai.GenerativeModel("gemini-1.5-flash")
+
+
+def generate_content(prompt):
+
+    response = model.generate_content(prompt)
+
+    return response.text

app/prompt.py

@@ -1,12 +1,12 @@
 import json
 import os
 from enum import Enum
-from typing import List, Optional
+from typing import List
 
-import cohere
 import pydantic
+from jeannie import generate_content
 
-os.environ["COHERE_API_KEY"] = "<YOUR-"API"-KEY>"
+os.environ["COHERE_API_KEY"] = "niT0L5XZyimfgvLYhzviA3Xs11cHSNMjkwzzQ9OB"
 
 
 class Tags(str, Enum):
@@ -48,25 +48,14 @@ class Resume(pydantic.BaseModel):
 
 
 class Smart:
-    def __init__(self):
-        self.llm = cohere.ClientV2(api_key=os.environ["COHERE_API_KEY"])
-
     def answer(self, path, lib, html):
         if path == "releases_url":
             return self.find_release(lib["library_name"], html)
         elif path == "releases_doc_url":
             return self.create_resume(lib, html)
 
     def query(self, prompt):
-        return self.llm.chat(
-            model="command-r-plus-08-2024",
-            messages=[
-                {
-                    "role": "user",
-                    "content": prompt,
-                }
-            ],
-        )
+        return generate_content(prompt)
 
     def find_release(self, lib, html):
         prompt = f"""
@@ -120,6 +109,8 @@ def create_resume(self, lib, html):
         Não seja criativo na formatação, responda exatamente como o schema fornecido.
         Verifique se sua resposta está no formato JSON para que seja possível usar o comando json.loads do python, Caso não esteja, formate-a corretamente.
 
+        Sempre responda no idioma Português do Brasil
+
         response_format:
         {json.dumps(Resume.model_json_schema())}
         """
@@ -128,7 +119,6 @@ def create_resume(self, lib, html):
     def reply(self, prompt):
 
         response = self.query(prompt)
-        response = response.message.content[0].text
         response = response.replace("```json", "")
         response = response.replace("```", "")
         response = response.replace("\n", "")

requirements.txt

@@ -1,3 +1,3 @@
 babel==2.15.0
-cohere==5.13.3
-crawl4ai==0.4.247
+crawl4ai==0.4.247
+google-generativeai==0.8.4