Add Docker support with Dockerfile and docker-compose.yaml; restructure project files and update requirements

Author: dan5e3s6ares

.gitignore

@@ -3,6 +3,11 @@ __pycache__/
 *.py[cod]
 *$py.class
 
+#PynewFiles
+pynews*
+format.json
+export.py
+
 # C extensions
 *.so
 

Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+    python3-pip \
+    python3.10 \
+    tzdata && \
+    ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime && \
+    dpkg-reconfigure --frontend noninteractive tzdata && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* &&\
+    mkdir -p app && \
+    pip3 install -U crawl4ai && \
+    playwright install && \
+    playwright install-deps
+
+WORKDIR /app
+
+COPY ./app/. .
+COPY requirements.txt .
+
+RUN pip3 install -r requirements.txt

README.md

@@ -6,33 +6,32 @@ Este Scraper utiliza IA para que não seja necessário a sua atualização const
 
 # Como utilizar
 
-Para criar um ambiente virtual, execute
-```
-python -m venv .venv
-```
+## Instalação do Docker e Docker Compose
 
-Para ativar o ambiente virtual, execute
-```
-source .venv/bin/activate
-```
+### Ubuntu
 
-Para instalar as dependências do projeto no ambiente virtual, execute
-```
-pip install -r requirements.txt
-```
+1. **Instale o Docker:**
 
-Para completar a instalação do Crawl4AI
-```
-# Install the package
-pip install -U crawl4ai
+    https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-22-04
 
-# Run post-installation setup
-crawl4ai-setup
+2. **Instale o Docker Compose:**
+
+    https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-compose-on-ubuntu-22-04
+
+
+### Windows
+
+1. **Instale o Docker:**
+
+    https://docs.docker.com/desktop/setup/install/windows-install/
+
+2. **Instale o Docker Compose:**
+
+    https://docs.docker.com/compose/install/
+
+
+### Crie uma chave junto a COHERE
 
-# Verify your installation
-crawl4ai-doctor
-```
-Crie uma chave junto a COHERE
 - https://docs.cohere.com/
 <p>
 
@@ -42,20 +41,23 @@ Atualize o valor em prompt.py
 os.environ["COHERE_API_KEY"] = "<YOUR-API-KEY>"
 ...
 ```
-Instale o Thinker
-```cmd
-sudo apt-get install python3-tk
+
+## Construa o container
+```sh
+docker compose build
 ```
 
-Para procurar por releases, execute o comando abaixo e aguarde as instruções
+## Para procurar por releases, execute o comando abaixo e aguarde as instruções
 ```
-python getNews.py releases
+docker compose run --rm --remove-orphans pynews python3 /app/app/getNews.py releases
 ```
 
-Para criar os resumos, execute
+## Para criar os resumos, execute
 ```
-python getNews.py slides
+docker compose run --rm --remove-orphans pynews python3 /app/app/getNews.py slides
 ```
+<p>
+<p>
+<p>
 
-
-Para desativar o ambiente virtual, execute `deactivate`.
+## Esse é um script que depende da COHERE AI, essa IA assim como todas as outras, ainda não apresentam comportamento estável, portanto esse script deve ser usado sob supervisão

app/__init__.py

@@ -3,126 +3,151 @@
         "library_name": "Requests",
         "releases_url": "https://pypi.org/project/requests/",
         "logo": "https://requests.readthedocs.io/en/latest/_static/requests-sidebar.png",
+        "repository": "https://github.com/psf/requests",
     },
     "Scikit-learn": {
         "library_name": "Scikit-learn",
         "releases_url": "https://pypi.org/project/scikit-learn/",
         "logo": "https://scikit-learn.org/stable/_static/scikit-learn-logo-small.png",
+        "repository": "https://github.com/scikit-learn/scikit-learn",
     },
     "Numpy": {
         "library_name": "Numpy",
         "releases_url": "https://pypi.org/project/numpy/",
         "logo": "https://numpy.org/devdocs/_static/numpylogo.svg",
+        "repository": "https://github.com/numpy/numpy",
     },
     "MatPlotLib": {
         "library_name": "MatPlotLib",
         "releases_url": "https://pypi.org/project/matplotlib/",
         "logo": "https://matplotlib.org/stable/_static/logo_light.svg",
+        "repository": "https://github.com/matplotlib/matplotlib",
     },
     "AIOHttp": {
         "library_name": "AIOHttp",
         "releases_url": "https://pypi.org/project/aiohttp/",
         "logo": "https://docs.aiohttp.org/en/stable/_static/aiohttp-plain.svg",
+        "repository": "https://github.com/aio-libs/aiohttp",
     },
     "Pandas": {
         "library_name": "Pandas",
         "releases_url": "https://pypi.org/project/pandas/",
         "logo": "https://pandas.pydata.org/static/img/pandas_mark.svg",
+        "repository": "https://github.com/pandas-dev/pandas",
     },
     "FastAPI": {
         "library_name": "FastAPI",
         "releases_url": "https://pypi.org/project/fastapi/",
-        "logo": "https://fastapi.tiangolo.com/img/icon.png",
+        "logo": "https://camo.githubusercontent.com/4ebb06d037b495f2c4c67e0ee4599f747e94e6323ece758a7da27fbbcb411250/68747470733a2f2f666173746170692e7469616e676f6c6f2e636f6d2f696d672f6c6f676f2d6d617267696e2f6c6f676f2d7465616c2e706e67",
+        "repository": "https://github.com/fastapi/fastapi",
     },
     "Django": {
         "library_name": "Django",
         "releases_url": "https://pypi.org/project/Django/",
         "logo": "https://static.djangoproject.com/img/logos/django-logo-negative.png",
+        "repository": "https://github.com/django/django",
     },
     "Seaborn": {
         "library_name": "Seaborn",
         "releases_url": "https://pypi.org/project/seaborn/",
         "logo": "https://seaborn.pydata.org/_images/logo-wide-lightbg.svg",
+        "repository": "https://github.com/mwaskom/seaborn",
     },
     "TensorFlow": {
         "library_name": "TensorFlow",
         "releases_url": "https://pypi.org/project/tensorflow/",
         "logo": "https://www.tensorflow.org/images/tf_logo_social.png",
+        "repository": "https://github.com/tensorflow/tensorflow",
     },
     "Keras": {
         "library_name": "Keras",
         "releases_url": "https://pypi.org/project/keras/",
         "logo": "https://keras.io/img/logo.png",
+        "repository": "https://github.com/keras-team/keras",
     },
     "PyTorch": {
         "library_name": "PyTorch",
         "releases_url": "https://pypi.org/project/torch/",
         "logo": "https://pytorch.org/assets/images/pytorch-logo.png",
+        "repository": "https://github.com/pytorch/pytorch",
     },
     "SQLAlchemy": {
         "library_name": "SQLAlchemy",
         "releases_url": "https://pypi.org/project/SQLAlchemy/",
         "logo": "https://www.sqlalchemy.org/img/sqla_logo.png",
+        "repository": "https://github.com/sqlalchemy/sqlalchemy",
     },
     "BeaultifulSoup": {
         "library_name": "BeaultifulSoup",
         "releases_url": "https://pypi.org/project/beautifulsoup4/",
         "logo": "https://www.crummy.com/software/BeautifulSoup/10.1.jpg",
+        "repository": None,
     },
     "LangChain": {
         "library_name": "LangChain",
         "releases_url": "https://pypi.org/project/langchain/",
         "logo": "https://python.langchain.com/img/brand/wordmark-dark.png",
+        "repository": "https://github.com/langchain-ai/langchain",
     },
     "CrewAI": {
         "library_name": "CrewAI",
         "releases_url": "https://pypi.org/project/crewai/",
         "logo": "https://cdn.prod.website-files.com/66cf2bfc3ed15b02da0ca770/66d07240057721394308addd_Logo%20(1).svg",
+        "repository": "https://github.com/crewAIInc/crewAI",
     },
     "Flask": {
         "library_name": "Flask",
         "releases_url": "https://pypi.org/project/Flask/",
         "logo": "https://flask.palletsprojects.com/en/stable/_static/flask-vertical.png",
+        "repository": "https://github.com/pallets/flask",
     },
     "Pygame": {
         "library_name": "Pygame",
         "releases_url": "https://pypi.org/project/pygame/",
         "logo": "https://www.pygame.org/images/logo_lofi.png",
+        "repository": "https://github.com/pygame/pygame",
     },
     "Thinker": {
         "library_name": "Thinker",
         "releases_url": "https://pypi.org/project/thinker/",
         "logo": "https://keras.io/img/logo.png",
+        "repository": "https://github.com/mehmetkose/thinker",
     },
     "Plotly": {
         "library_name": "Plotly",
         "releases_url": "https://pypi.org/project/plotly/",
         "logo": "https://plotly.com/static/img/logos/plotly-logomark.svg",
+        "repository": "https://github.com/plotly/plotly.py",
     },
     "MlForecast": {
         "library_name": "MlForecast",
         "releases_url": "https://pypi.org/project/mlforecast/",
         "logo": "https://raw.githubusercontent.com/Nixtla/mlforecast/main/nbs/figs/logo.png",
+        "repository": "https://github.com/Nixtla/mlforecast",
     },
     "GeoPandas": {
         "library_name": "GeoPandas",
         "releases_url": "https://pypi.org/project/geopandas/",
         "logo": "https://geopandas.org/en/stable/_static/geopandas_logo_web.svg",
+        "repository": "https://github.com/geopandas/geopandas",
     },
     "AirFlow": {
         "library_name": "AirFlow",
         "releases_url": "https://pypi.org/project/apache-airflow/",
         "logo": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/de/AirflowLogo.png/800px-AirflowLogo.png?20191014185111",
+        "repository": "https://github.com/apache/airflow",
     },
     "PySpark": {
         "library_name": "PySpark",
         "releases_url": "https://pypi.org/project/pyspark/",
         "logo": "https://spark.apache.org/docs/latest/api/python/_static/spark-logo-reverse.png",
+        "repository": "https://github.com/apache/spark/tree/master/python",
     },
     "Gym": {
         "library_name": "Gym",
         "releases_url": "https://pypi.org/project/gym/",
         "logo": "https://www.gymlibrary.dev/_static/img/gym_logo_black.svg",
+        "repository": "https://github.com/Farama-Foundation/Gymnasium",
     },
     "HyperOpt": {
         "library_name": "HyperOpt",
@@ -139,4 +164,10 @@
         "releases_url": "https://crawl4ai.com/mkdocs/blog/",
         "logo": "https://star-history.com/#unclecode/crawl4ai&Date",
     },
+    "ScanAPI": {
+        "library_name": "ScanAPI",
+        "releases_url": "https://pypi.org/project/scanapi/",
+        "logo": "https://avatars.githubusercontent.com/u/59395469?s=200&v=4",
+        "repository": "https://github.com/scanapi/scanapi",
+    },
 }

bibliotecas.py app/bibliotecas.py

@@ -4,8 +4,9 @@
 import time
 from datetime import datetime
 from json import dump
-from tkinter import messagebox
 
+from bibliotecas import bibliotecas
+from cacheVariables import pynews
 from crawl4ai import (
     AsyncWebCrawler,
     BrowserConfig,
@@ -14,9 +15,6 @@
     DefaultMarkdownGenerator,
     PruningContentFilter,
 )
-
-from bibliotecas import bibliotecas
-from cacheVariables import pynews
 from prompt import Smart
 
 md_generator = DefaultMarkdownGenerator(
@@ -60,7 +58,16 @@ async def fetch(self, lib, url_name):
             )
             new_html = html.markdown_v2
 
-            response = Smart().answer(url_name, lib, new_html.fit_markdown)
+            response = {}
+            try:
+                response = Smart().answer(url_name, lib, new_html.fit_markdown)
+            except Exception:
+                print("$$$ COHERE AI Time Out $$$")
+                print("Biblioteca não processada :", lib["library_name"])
+                print(
+                    "Tente rodar o script novamente somente com a lib : ",
+                    lib["library_name"],
+                )
 
             release_date = response.get("release_date")
 
@@ -113,12 +120,6 @@ async def main():
         f"\033[1;37;44m\033[1;30m\n Edite o arquivo {news_json_file} \
 para adicionar as urls contendo o descritivo das novas releases \n"
     )
-    messagebox.showwarning(
-        "Aviso",
-        f"Edite o arquivo {news_json_file} para adicionar as urls contendo \
-o descritivo das novas releases",
-        icon="warning",
-    )
 
 
 async def slides():
@@ -127,8 +128,20 @@ async def slides():
     with open(news_json_file, "r", encoding="utf-8") as f:
         news.study_case = json.load(f)
     await news.get("releases_doc_url")
-    with open(news_json_file, "w", encoding="utf-8") as f:
-        dump(pynews, f)
+    slides = {}
+    try:
+        with open("pynews_slides.json", "r", encoding="utf-8") as f:
+            slides = json.load(f)
+    except Exception:
+        pass
+    for item in pynews:
+        slides[item] = pynews[item]
+    with open("pynews_slides.json", "w", encoding="utf-8") as f:
+        dump(slides, f)
+    print(
+        "\033[1;37;44m\033[1;30m\n Abra o arquivo pynews_slides.json para \
+ter acesso ao conteúdo produzido.\n"
+    )
 
 
 if __name__ == "__main__":