Merge pull request #29 from igorbenav/more-efficient-query

More efficient select

Author: igorbenav

README.md

@@ -47,6 +47,7 @@
 - Easy redis caching
 - Easy client-side caching
 - ARQ integration for task queue
+- Efficient querying (only queries what's needed)
 - Easily extendable
 - Flexible
 - Easy running with docker compose
@@ -63,6 +64,7 @@
 #### Features
 - [ ] Add a Rate Limiter decorator
 - [ ] Add mongoDB support
+- [ ] Add support in schema_to_select for dict as well as Pydantic Schema
 
 #### Security
 - [x] FastAPI docs behind authentication and hidden based on the environment
@@ -403,12 +405,19 @@ First, you may want to take a look at the project structure and understand what
 └── src                               # Source code directory.
     ├── __init__.py                   # Initialization file for the src package.
     ├── alembic.ini                   # Configuration file for Alembic (database migration tool).
+    ├── poetry.lock
+    ├── pyproject.toml                # Configuration file for Poetry, lists project dependencies.
+    │
     ├── app                           # Main application directory.
     │   ├── __init__.py               # Initialization file for the app package.
+    │   ├── main.py                   # Entry point that imports and creates the FastAPI application instance.
+    │   ├── worker.py                 # Worker script for handling background tasks.
+    │   │
     │   ├── api                       # Folder containing API-related logic.
     │   │   ├── __init__.py
     │   │   ├── dependencies.py       # Defines dependencies that can be reused across the API endpoints.
     │   │   ├── exceptions.py         # Contains custom exceptions for the API.
+    │   │   │
     │   │   └── v1                    # Version 1 of the API.
     │   │       ├── __init__.py
     │   │       ├── login.py          # API routes related to user login.
@@ -431,37 +440,38 @@ First, you may want to take a look at the project structure and understand what
     │   │   ├── __init__.py
     │   │   ├── crud_base.py          # Base CRUD operations class that can be extended by other CRUD modules.
     │   │   ├── crud_posts.py         # CRUD operations for posts.
-    │   │   └── crud_users.py         # CRUD operations for users.
+    │   │   ├── crud_users.py         # CRUD operations for users.
+    │   │   └── helper.py             # Helper functions for CRUD operations.
     │   │
-    │   ├── main.py                   # Entry point that imports and creates the FastAPI application instance.
     │   ├── models                    # ORM models for the application.
     │   │   ├── __init__.py
     │   │   ├── post.py               # ORM model for posts.
     │   │   └── user.py               # ORM model for users.
     │   │
-    │   ├── schemas                   # Pydantic schemas for data validation.
-    │   │   ├── __init__.py
-    │   │   ├── job.py                # Schemas related to background jobs.
-    │   │   ├── post.py               # Schemas related to posts.
-    │   │   └── user.py               # Schemas related to users.
-    │   │
-    │   └── worker.py                 # Worker script for handling background tasks.
+    │   └── schemas                   # Pydantic schemas for data validation.
+    │       ├── __init__.py
+    │       ├── job.py                # Schemas related to background jobs.
+    │       ├── post.py               # Schemas related to posts.
+    │       └── user.py               # Schemas related to users.
     │
     ├── migrations                    # Directory for Alembic migrations.
     │   ├── README                    # General info and guidelines for migrations.
     │   ├── env.py                    # Environment configurations for Alembic.
     │   ├── script.py.mako            # Template script for migration generation.
+    │   │
     │   └── versions                  # Folder containing individual migration scripts.
+    │       └── README.MD
     │
-    ├── pyproject.toml                # Configuration file for Poetry, lists project dependencies.
     ├── scripts                       # Utility scripts for the project.
+    │   ├── __init__.py
     │   └── create_first_superuser.py # Script to create the first superuser in the application.
     │
     └── tests                         # Directory containing all the tests.
         ├── __init__.py               # Initialization file for the tests package.
         ├── conftest.py               # Configuration and fixtures for pytest.
         ├── helper.py                 # Helper functions for writing tests.
         └── test_user.py              # Tests related to the user model and endpoints.
+
 ```
 
 ### 5.2 Database Model
@@ -552,6 +562,89 @@ CRUDEntity = CRUDBase[Entity, EntityCreateInternal, EntityUpdate, EntityUpdateIn
 crud_entity = CRUDEntity(Entity)
 ```
 
+So, for users:
+```python
+# crud_users.py
+from app.model.user import User
+from app.schemas.user import UserCreateInternal, UserUpdate, UserUpdateInternal, UserDelete
+
+CRUDUser = CRUDBase[User, UserCreateInternal, UserUpdate, UserUpdateInternal, UserDelete]
+crud_users = CRUDUser(User)
+```
+
+When actually using the crud in an endpoint, to get data you just pass the database connection and the attributes as kwargs:
+```python
+# Here I'm getting the users with email == user.email
+user = await crud_users.get(db=db, email=user.email)
+```
+
+To get a list of objects with the attributes, you should use the get_multi:
+```python
+# Here I'm getting 100 users with the name David except for the first 3
+user = await crud_users.get_multi(
+  db=db,
+  offset=3,
+  limit=100,
+  name="David"
+)
+```
+
+To create, you pass a `CreateSchemaType` object with the attributes, such as a `UserCreate` pydantic schema:
+```python
+from app.core.schemas.user import UserCreate
+
+# Creating the object
+user_internal = UserCreate(
+  name="user",
+  username="myusername",
+  email="[email protected]"
+)
+
+# Passing the object to be created
+crud_users.create(db=db, object=user_internal)
+```
+
+To just check if there is at least one row that matches a certain set of attributes, you should use `exists`
+```python
+# This queries only the email variable
+# It returns True if there's at least one or False if there is none
+crud_users.exists(db=db, email=user@example.com)
+```
+
+To update you pass an `object` which may be a `pydantic schema` or just a regular `dict`, and the kwargs.
+You will update with `objects` the rows that match your `kwargs`.
+```python
+# Here I'm updating the user with username == "myusername". 
+# #I'll change his name to "Updated Name"
+crud_users.update(db=db, object={name="Updated Name"}, username="myusername")
+```
+
+To delete we have two options:
+- db_delete: actually deletes the row from the database
+- delete: 
+    - adds `"is_deleted": True` and `deleted_at: datetime.utcnow()` if the model inherits from `PersistentDeletion` (performs a soft delete), but keeps the object in the database.
+    - actually deletes the row from the database if the model does not inherit from `PersistentDeletion`
+
+```python
+# Here I'll just change is_deleted to True
+crud_users.delete(db=db, username="myusername")
+
+# Here I actually delete it from the database
+crud_users.db_delete(db=db, username="myusername")
+```
+
+#### More Efficient Selecting
+For the `get` and `get_multi` methods we have the option to define a `schema_to_select` attribute, which is what actually makes the queries more efficient. When you pass a pydantic schema in `schema_to_select` to the `get` or `get_multi` methods, only the attributes in the schema will be selected.
+```python
+from app.schemas.user import UserRead
+# Here it's selecting all of the user's data
+crud_user.get(db=db, username="myusername")
+
+# Now it's only selecting the data that is in UserRead. 
+# Since that's my response_model, it's all I need
+crud_user.get(db=db, username="myusername", schema_to_select=UserRead)
+```
+
 ### 5.7 Routes
 Inside `app/api/v1`, create a new `entities.py` file and create the desired routes
 ```python

src/app/api/v1/posts.py

@@ -23,7 +23,7 @@ async def write_post(
     current_user: Annotated[UserRead, Depends(get_current_user)],
     db: Annotated[AsyncSession, Depends(async_get_db)]
 ):
-    db_user = await crud_users.get(db=db, username=username, is_deleted=False)
+    db_user = await crud_users.get(db=db, schema_to_select=UserRead, username=username, is_deleted=False)
     if db_user is None:
         raise HTTPException(status_code=404, detail="User not found")
 
@@ -32,6 +32,7 @@ async def write_post(
 
     post_internal_dict = post.model_dump()
     post_internal_dict["created_by_user_id"] = db_user.id
+
     post_internal = PostCreateInternal(**post_internal_dict)
     return await crud_posts.create(db=db, object=post_internal)
 
@@ -43,11 +44,11 @@ async def read_posts(
     username: str, 
     db: Annotated[AsyncSession, Depends(async_get_db)]
 ):
-    db_user = await crud_users.get(db=db, username=username, is_deleted=False)
+    db_user = await crud_users.get(db=db, schema_to_select=UserRead, username=username, is_deleted=False)
     if db_user is None:
         raise HTTPException(status_code=404, detail="User not found")
 
-    posts = await crud_posts.get_multi(db=db, created_by_user_id=db_user.id, is_deleted=False)
+    posts = await crud_posts.get_multi(db=db, schema_to_select=PostRead, created_by_user_id=db_user.id, is_deleted=False)
     return posts
 
 
@@ -59,18 +60,18 @@ async def read_post(
     id: int, 
     db: Annotated[AsyncSession, Depends(async_get_db)]
 ):
-    db_user = await crud_users.get(db=db, username=username, is_deleted=False)
+    db_user = await crud_users.get(db=db, schema_to_select=UserRead, username=username, is_deleted=False)
     if db_user is None:
         raise HTTPException(status_code=404, detail="User not found")
 
-    db_post = await crud_posts.get(db=db, id=id, created_by_user_id=db_user.id, is_deleted=False)
+    db_post = await crud_posts.get(db=db, schema_to_select=PostRead, id=id, created_by_user_id=db_user.id, is_deleted=False)
     if db_post is None:
         raise HTTPException(status_code=404, detail="Post not found")
 
     return db_post
 
 
-@router.patch("/{username}/post/{id}", response_model=PostRead)
+@router.patch("/{username}/post/{id}")
 @cache(
     "{username}_post_cache", 
     resource_id_name="id", 
@@ -84,18 +85,19 @@ async def patch_post(
     current_user: Annotated[UserRead, Depends(get_current_user)],
     db: Annotated[AsyncSession, Depends(async_get_db)]
 ):
-    db_user = await crud_users.get(db=db, username=username, is_deleted=False)
+    db_user = await crud_users.get(db=db, schema_to_select=UserRead, username=username, is_deleted=False)
     if db_user is None:
         raise HTTPException(status_code=404, detail="User not found")
 
     if current_user.id != db_user.id:
         raise privileges_exception
 
-    db_post = await crud_posts.get(db=db, id=id, is_deleted=False)
+    db_post = await crud_posts.get(db=db, schema_to_select=PostRead, id=id, is_deleted=False)
     if db_post is None:
         raise HTTPException(status_code=404, detail="Post not found")
 
-    return await crud_posts.update(db=db, object=values, db_object=db_post, id=id)
+    await crud_posts.update(db=db, object=values, id=id)
+    return {"message": "Post updated"}
 
 
 @router.delete("/{username}/post/{id}")
@@ -111,24 +113,20 @@ async def erase_post(
     current_user: Annotated[UserRead, Depends(get_current_user)],
     db: Annotated[AsyncSession, Depends(async_get_db)]
 ):
-    db_user = await crud_users.get(db=db, username=username, is_deleted=False)
+    db_user = await crud_users.get(db=db, schema_to_select=UserRead, username=username, is_deleted=False)
     if db_user is None:
         raise HTTPException(status_code=404, detail="User not found")
 
     if current_user.id != db_user.id:
         raise privileges_exception
 
-    db_post = await crud_posts.get(db=db, id=id, is_deleted=False)
+    db_post = await crud_posts.get(db=db, schema_to_select=PostRead, id=id, is_deleted=False)
     if db_post is None:
         raise HTTPException(status_code=404, detail="Post not found")
 
-    deleted_post = await crud_posts.delete(db=db, db_object=db_post, id=id)
-    if deleted_post.is_deleted == True:
-        message = {"message": "Post deleted"}
-    else:
-        message = {"message": "Something went wrong"}
-
-    return message
+    await crud_posts.delete(db=db, db_row=db_post, id=id)
+    
+    return {"message": "Post deleted"}
 
 
 @router.delete("/{username}/db_post/{id}", dependencies=[Depends(get_current_superuser)])
@@ -143,20 +141,13 @@ async def erase_db_post(
     id: int,
     db: Annotated[AsyncSession, Depends(async_get_db)]
 ):
-    db_user = await crud_users.get(db=db, username=username, is_deleted=False)
+    db_user = await crud_users.get(db=db, schema_to_select=UserRead, username=username, is_deleted=False)
     if db_user is None:
         raise HTTPException(status_code=404, detail="User not found")
 
-    db_post = await crud_posts.get(db=db, id=id, is_deleted=False)
+    db_post = await crud_posts.get(db=db, schema_to_select=PostRead, id=id, is_deleted=False)
     if db_post is None:
         raise HTTPException(status_code=404, detail="Post not found")
 
     await crud_posts.db_delete(db=db, db_object=db_post, id=id)
-    deleted_post = await crud_posts.get(db=db, id=id)
-    
-    if deleted_post is None:
-        message = {"message": "Post deleted"}
-    else:
-        message = {"message": "Something went wrong"}
-
-    return message
+    return {"message": "Post deleted from the database"}