Add Docker support with Dockerfile and docker-compose.yaml; restructure project files and update requirements

Author: dan5e3s6ares

Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+    python3-pip \
+    python3.10 \
+    tzdata && \
+    ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime && \
+    dpkg-reconfigure --frontend noninteractive tzdata && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* &&\
+    mkdir -p app && \
+    pip3 install -U crawl4ai && \
+    playwright install && \
+    playwright install-deps
+
+WORKDIR /app
+
+COPY ./app/. .
+COPY requirements.txt .
+
+RUN pip3 install -r requirements.txt

README.md

@@ -6,33 +6,32 @@ Este Scraper utiliza IA para que não seja necessário a sua atualização const
 
 # Como utilizar
 
-Para criar um ambiente virtual, execute
-```
-python -m venv .venv
-```
+## Instalação do Docker e Docker Compose
 
-Para ativar o ambiente virtual, execute
-```
-source .venv/bin/activate
-```
+### Ubuntu
 
-Para instalar as dependências do projeto no ambiente virtual, execute
-```
-pip install -r requirements.txt
-```
+1. **Instale o Docker:**
 
-Para completar a instalação do Crawl4AI
-```
-# Install the package
-pip install -U crawl4ai
+    https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-22-04
 
-# Run post-installation setup
-crawl4ai-setup
+2. **Instale o Docker Compose:**
+
+    https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-compose-on-ubuntu-22-04
+
+
+### Windows
+
+1. **Instale o Docker:**
+
+    https://docs.docker.com/desktop/setup/install/windows-install/
+
+2. **Instale o Docker Compose:**
+
+    https://docs.docker.com/compose/install/
+
+
+### Crie uma chave junto a COHERE
 
-# Verify your installation
-crawl4ai-doctor
-```
-Crie uma chave junto a COHERE
 - https://docs.cohere.com/
 <p>
 
@@ -42,20 +41,23 @@ Atualize o valor em prompt.py
 os.environ["COHERE_API_KEY"] = "<YOUR-API-KEY>"
 ...
 ```
-Instale o Thinker
-```cmd
-sudo apt-get install python3-tk
+
+## Construa o container
+```sh
+docker compose build
 ```
 
-Para procurar por releases, execute o comando abaixo e aguarde as instruções
+## Para procurar por releases, execute o comando abaixo e aguarde as instruções
 ```
-python getNews.py releases
+docker compose run --rm --remove-orphans pynews python3 /app/app/getNews.py releases
 ```
 
-Para criar os resumos, execute
+## Para criar os resumos, execute
 ```
-python getNews.py slides
+docker compose run --rm --remove-orphans pynews python3 /app/app/getNews.py slides
 ```
+<p>
+<p>
+<p>
 
-
-Para desativar o ambiente virtual, execute `deactivate`.
+## Esse é um script que depende da COHERE AI, essa IA assim como todas as outras, ainda não apresentam comportamento estável, portanto esse script deve ser usado sob supervisão

app/__init__.py

@@ -3,126 +3,151 @@
         "library_name": "Requests",
         "releases_url": "https://pypi.org/project/requests/",
         "logo": "https://requests.readthedocs.io/en/latest/_static/requests-sidebar.png",
+        "repository": "https://github.com/psf/requests",
     },
     "Scikit-learn": {
         "library_name": "Scikit-learn",
         "releases_url": "https://pypi.org/project/scikit-learn/",
         "logo": "https://scikit-learn.org/stable/_static/scikit-learn-logo-small.png",
+        "repository": "https://github.com/scikit-learn/scikit-learn",
     },
     "Numpy": {
         "library_name": "Numpy",
         "releases_url": "https://pypi.org/project/numpy/",
         "logo": "https://numpy.org/devdocs/_static/numpylogo.svg",
+        "repository": "https://github.com/numpy/numpy",
     },
     "MatPlotLib": {
         "library_name": "MatPlotLib",
         "releases_url": "https://pypi.org/project/matplotlib/",
         "logo": "https://matplotlib.org/stable/_static/logo_light.svg",
+        "repository": "https://github.com/matplotlib/matplotlib",
     },
     "AIOHttp": {
         "library_name": "AIOHttp",
         "releases_url": "https://pypi.org/project/aiohttp/",
         "logo": "https://docs.aiohttp.org/en/stable/_static/aiohttp-plain.svg",
+        "repository": "https://github.com/aio-libs/aiohttp",
     },
     "Pandas": {
         "library_name": "Pandas",
         "releases_url": "https://pypi.org/project/pandas/",
         "logo": "https://pandas.pydata.org/static/img/pandas_mark.svg",
+        "repository": "https://github.com/pandas-dev/pandas",
     },
     "FastAPI": {
         "library_name": "FastAPI",
         "releases_url": "https://pypi.org/project/fastapi/",
         "logo": "https://fastapi.tiangolo.com/img/icon.png",
+        "repository": "https://github.com/fastapi/fastapi",
     },
     "Django": {
         "library_name": "Django",
         "releases_url": "https://pypi.org/project/Django/",
         "logo": "https://static.djangoproject.com/img/logos/django-logo-negative.png",
+        "repository": "https://github.com/django/django",
     },
     "Seaborn": {
         "library_name": "Seaborn",
         "releases_url": "https://pypi.org/project/seaborn/",
         "logo": "https://seaborn.pydata.org/_images/logo-wide-lightbg.svg",
+        "repository": "https://github.com/mwaskom/seaborn",
     },
     "TensorFlow": {
         "library_name": "TensorFlow",
         "releases_url": "https://pypi.org/project/tensorflow/",
         "logo": "https://www.tensorflow.org/images/tf_logo_social.png",
+        "repository": "https://github.com/tensorflow/tensorflow",
     },
     "Keras": {
         "library_name": "Keras",
         "releases_url": "https://pypi.org/project/keras/",
         "logo": "https://keras.io/img/logo.png",
+        "repository": "https://github.com/keras-team/keras",
     },
     "PyTorch": {
         "library_name": "PyTorch",
         "releases_url": "https://pypi.org/project/torch/",
         "logo": "https://pytorch.org/assets/images/pytorch-logo.png",
+        "repository": "https://github.com/pytorch/pytorch",
     },
     "SQLAlchemy": {
         "library_name": "SQLAlchemy",
         "releases_url": "https://pypi.org/project/SQLAlchemy/",
         "logo": "https://www.sqlalchemy.org/img/sqla_logo.png",
+        "repository": "https://github.com/sqlalchemy/sqlalchemy",
     },
     "BeaultifulSoup": {
         "library_name": "BeaultifulSoup",
         "releases_url": "https://pypi.org/project/beautifulsoup4/",
         "logo": "https://www.crummy.com/software/BeautifulSoup/10.1.jpg",
+        "repository": None,
     },
     "LangChain": {
         "library_name": "LangChain",
         "releases_url": "https://pypi.org/project/langchain/",
         "logo": "https://python.langchain.com/img/brand/wordmark-dark.png",
+        "repository": "https://github.com/langchain-ai/langchain",
     },
     "CrewAI": {
         "library_name": "CrewAI",
         "releases_url": "https://pypi.org/project/crewai/",
         "logo": "https://cdn.prod.website-files.com/66cf2bfc3ed15b02da0ca770/66d07240057721394308addd_Logo%20(1).svg",
+        "repository": "https://github.com/crewAIInc/crewAI",
     },
     "Flask": {
         "library_name": "Flask",
         "releases_url": "https://pypi.org/project/Flask/",
         "logo": "https://flask.palletsprojects.com/en/stable/_static/flask-vertical.png",
+        "repository": "https://github.com/pallets/flask",
     },
     "Pygame": {
         "library_name": "Pygame",
         "releases_url": "https://pypi.org/project/pygame/",
         "logo": "https://www.pygame.org/images/logo_lofi.png",
+        "repository": "https://github.com/pygame/pygame",
     },
     "Thinker": {
         "library_name": "Thinker",
         "releases_url": "https://pypi.org/project/thinker/",
         "logo": "https://keras.io/img/logo.png",
+        "repository": "https://github.com/mehmetkose/thinker",
     },
     "Plotly": {
         "library_name": "Plotly",
         "releases_url": "https://pypi.org/project/plotly/",
         "logo": "https://plotly.com/static/img/logos/plotly-logomark.svg",
+        "repository": "https://github.com/plotly/plotly.py",
     },
     "MlForecast": {
         "library_name": "MlForecast",
         "releases_url": "https://pypi.org/project/mlforecast/",
         "logo": "https://raw.githubusercontent.com/Nixtla/mlforecast/main/nbs/figs/logo.png",
+        "repository": "https://github.com/Nixtla/mlforecast",
     },
     "GeoPandas": {
         "library_name": "GeoPandas",
         "releases_url": "https://pypi.org/project/geopandas/",
         "logo": "https://geopandas.org/en/stable/_static/geopandas_logo_web.svg",
+        "repository": "https://github.com/geopandas/geopandas",
     },
     "AirFlow": {
         "library_name": "AirFlow",
         "releases_url": "https://pypi.org/project/apache-airflow/",
         "logo": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/de/AirflowLogo.png/800px-AirflowLogo.png?20191014185111",
+        "repository": "https://github.com/apache/airflow",
     },
     "PySpark": {
         "library_name": "PySpark",
         "releases_url": "https://pypi.org/project/pyspark/",
         "logo": "https://spark.apache.org/docs/latest/api/python/_static/spark-logo-reverse.png",
+        "repository": "https://github.com/apache/spark/tree/master/python",
     },
     "Gym": {
         "library_name": "Gym",
         "releases_url": "https://pypi.org/project/gym/",
         "logo": "https://www.gymlibrary.dev/_static/img/gym_logo_black.svg",
+        "repository": "https://github.com/Farama-Foundation/Gymnasium",
     },
     "HyperOpt": {
         "library_name": "HyperOpt",

bibliotecas.py app/bibliotecas.py

@@ -4,8 +4,9 @@
 import time
 from datetime import datetime
 from json import dump
-from tkinter import messagebox
 
+from bibliotecas import bibliotecas
+from cacheVariables import pynews
 from crawl4ai import (
     AsyncWebCrawler,
     BrowserConfig,
@@ -14,9 +15,6 @@
     DefaultMarkdownGenerator,
     PruningContentFilter,
 )
-
-from bibliotecas import bibliotecas
-from cacheVariables import pynews
 from prompt import Smart
 
 md_generator = DefaultMarkdownGenerator(
@@ -60,7 +58,16 @@ async def fetch(self, lib, url_name):
             )
             new_html = html.markdown_v2
 
-            response = Smart().answer(url_name, lib, new_html.fit_markdown)
+            response = {}
+            try:
+                response = Smart().answer(url_name, lib, new_html.fit_markdown)
+            except Exception:
+                print("$$$ COHERE AI Time Out $$$")
+                print("Biblioteca não processada :", lib["library_name"])
+                print(
+                    "Tente rodar o script novamente somente com a lib : ",
+                    lib["library_name"],
+                )
 
             release_date = response.get("release_date")
 
@@ -113,12 +120,6 @@ async def main():
         f"\033[1;37;44m\033[1;30m\n Edite o arquivo {news_json_file} \
 para adicionar as urls contendo o descritivo das novas releases \n"
     )
-    messagebox.showwarning(
-        "Aviso",
-        f"Edite o arquivo {news_json_file} para adicionar as urls contendo \
-o descritivo das novas releases",
-        icon="warning",
-    )
 
 
 async def slides():

cacheVariables.py app/cacheVariables.py

@@ -6,7 +6,7 @@
 import cohere
 import pydantic
 
-os.environ["COHERE_API_KEY"] = "" < YOUR - API - KEY > ""
+os.environ["COHERE_API_KEY"] = "<YOUR-"API"-KEY>"
 
 
 class Tags(str, Enum):
@@ -116,20 +116,32 @@ def create_resume(self, lib, html):
             - Não é necessário preencher todas as categorias, apenas as que possuem informações.
             - Se a informação não estiver disponível, retorne None para todos os campos, exceto para `library_name`.
 
-        Schema Pydantic para a resposta:
-        {Resume.model_json_schema()}
+        Não seja criativo, responda somente com que for encontrado no documento.
+        Não seja criativo na formatação, responda exatamente como o schema fornecido.
+        Verifique se sua resposta está no formato JSON para que seja possível usar o comando json.loads do python, Caso não esteja, formate-a corretamente.
+
+        response_format:
+        {json.dumps(Resume.model_json_schema())}
         """
         return self.reply(prompt)
 
     def reply(self, prompt):
 
-        response = self.query(prompt).message.content[0].text
+        response = self.query(prompt)
+        response = response.message.content[0].text
         response = response.replace("```json", "")
         response = response.replace("```", "")
         response = response.replace("\n", "")
-
-        try:
-            return json.loads(response)
-        except json.JSONDecodeError:
-            print("Erro ao decodificar a resposta JSON:", response)
-            return {}
+        _response = {}
+        one_time = True
+        while len(response) > 10:
+            try:
+                _response = json.loads(response)
+                break
+            except json.JSONDecodeError:
+                if one_time:
+                    print("### COHERE retornou JSON em formato errado")
+                    print("### Tentando formatar ###")
+                    one_time = False
+                response = response[:-1]
+        return _response

getNews.py app/getNews.py

@@ -0,0 +1,6 @@
+services:
+  pynews:
+    build: .
+    volumes:
+      - .:/app
+    working_dir: /app

prompt.py app/prompt.py

@@ -0,0 +1,50 @@
+{
+    "Scikit-learn": {
+        "version": "1.6.1",
+        "release_date": "2025-01-10",
+        "releases_doc_url": "https://scikit-learn.org/stable/whats_new/v1.6.html#release-notes-1-6",
+        "library_name": "Scikit-learn"
+    },
+    "FastAPI": {
+        "version": "0.115.7",
+        "release_date": "2025-01-22",
+        "releases_doc_url": "https://fastapi.tiangolo.com/release-notes/",
+        "library_name": "FastAPI"
+    },
+    "Numpy": {
+        "version": "2.2.2",
+        "release_date": "2025-01-18",
+        "releases_doc_url": "https://github.com/numpy/numpy/releases/tag/v2.2.2",
+        "library_name": "Numpy"
+    },
+    "Django": {
+        "version": "5.1.5",
+        "release_date": "2025-01-14",
+        "releases_doc_url": "https://docs.djangoproject.com/en/dev/releases/5.1.5/",
+        "library_name": "Django"
+    },
+    "CrewAI": {
+        "version": "0.98.0",
+        "release_date": "2025-01-20",
+        "releases_doc_url": "https://github.com/crewAIInc/crewAI/releases",
+        "library_name": "CrewAI"
+    },
+    "Keras": {
+        "version": "3.8.0",
+        "release_date": "2025-01-07",
+        "releases_doc_url": "https://github.com/keras-team/keras/releases",
+        "library_name": "Keras"
+    },
+    "LangChain": {
+        "version": "0.3.15",
+        "release_date": "2025-01-20",
+        "releases_doc_url": "https://github.com/langchain-ai/langchain/releases/tag/langchain%3D%3D0.3.15",
+        "library_name": "LangChain"
+    },
+    "SQLAlchemy": {
+        "version": "2.0.37",
+        "release_date": "2025-01-09",
+        "releases_doc_url": "https://github.com/sqlalchemy/sqlalchemy/releases",
+        "library_name": "SQLAlchemy"
+    }
+}

docker-compose.yaml

@@ -1,5 +1,3 @@
-beautifulsoup4==4.12.3
 babel==2.15.0
 cohere==5.13.3
-crawl4ai==0.4.247
-tk==0.1.0
+crawl4ai==0.4.247