Merge branch 'update/document_module' into feature/doc-transform

Author: avirajsingh7

backend/app/alembic/versions/42c4c8b22a09_add_is_deleted_column_in_document.py

@@ -0,0 +1,54 @@
+"""Add is_deleted column in document
+
+Revision ID: 42c4c8b22a09
+Revises: 38f0e8c8dc92
+Create Date: 2025-08-22 15:16:29.489991
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel.sql.sqltypes
+
+
+# revision identifiers, used by Alembic.
+revision = "42c4c8b22a09"
+down_revision = "38f0e8c8dc92"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("document", sa.Column("project_id", sa.Integer(), nullable=False))
+    op.add_column("document", sa.Column("is_deleted", sa.Boolean(), nullable=False))
+
+    # Must Do  --> Think about how we will execute this migration on current DB
+    # Dropping Owner id, adding values to project id column and more idk rn
+
+    op.drop_constraint("document_owner_id_fkey", "document", type_="foreignkey")
+    op.create_foreign_key(
+        None, "document", "project", ["project_id"], ["id"], ondelete="CASCADE"
+    )
+    op.drop_column("document", "owner_id")
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "document",
+        sa.Column("owner_id", sa.INTEGER(), autoincrement=False, nullable=False),
+    )
+    op.drop_constraint(None, "document", type_="foreignkey")
+    op.create_foreign_key(
+        "document_owner_id_fkey",
+        "document",
+        "user",
+        ["owner_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+    op.drop_column("document", "is_deleted")
+    op.drop_column("document", "project_id")
+    # ### end Alembic commands ###

backend/app/alembic/versions/b79fc198879a_add_storage_path_column_in_project_table.py

@@ -0,0 +1,31 @@
+"""Add storage_path column in project table
+
+Revision ID: b79fc198879a
+Revises: 42c4c8b22a09
+Create Date: 2025-08-22 16:23:28.162616
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel.sql.sqltypes
+
+
+# revision identifiers, used by Alembic.
+revision = "b79fc198879a"
+down_revision = "42c4c8b22a09"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("project", sa.Column("storage_path", sa.Uuid(), nullable=False))
+    op.create_unique_constraint(None, "project", ["storage_path"])
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, "project", type_="unique")
+    op.drop_column("project", "storage_path")
+    # ### end Alembic commands ###

backend/app/api/routes/collections.py

@@ -24,7 +24,7 @@
     DocumentCollectionCrud,
 )
 from app.crud.rag import OpenAIVectorStoreCrud, OpenAIAssistantCrud
-from app.models import Collection, Document
+from app.models import Collection, Document, DocumentPublic
 from app.models.collection import CollectionStatus
 from app.utils import APIResponse, load_description, get_openai_client
 
@@ -225,8 +225,8 @@ def do_create_collection(
         else WebHookCallback(request.callback_url, payload)
     )
 
-    storage = AmazonCloudStorage(current_user)
-    document_crud = DocumentCrud(session, current_user.id)
+    storage = AmazonCloudStorage(current_user.project_id)
+    document_crud = DocumentCrud(session, current_user.project_id)
     assistant_crud = OpenAIAssistantCrud(client)
     vector_store_crud = OpenAIVectorStoreCrud(client)
     collection_crud = CollectionCrud(session, current_user.id)
@@ -423,7 +423,7 @@ def list_collections(
 @router.post(
     "/docs/{collection_id}",
     description=load_description("collections/docs.md"),
-    response_model=APIResponse[List[Document]],
+    response_model=APIResponse[List[DocumentPublic]],
 )
 def collection_documents(
     session: SessionDep,

backend/app/api/routes/documents.py

@@ -3,13 +3,13 @@
 from typing import List, Optional
 from pathlib import Path
 
-from fastapi import APIRouter, File, UploadFile, Query, Form, BackgroundTasks
+from fastapi import APIRouter, File, UploadFile, Query, Form, BackgroundTasks, HTTPException
 from fastapi import Path as FastPath
 from fastapi.responses import JSONResponse
 from fastapi import HTTPException
 
-from app.crud import DocumentCrud, CollectionCrud
-from app.models import Document
+from app.crud import DocumentCrud, CollectionCrud, get_project_by_id
+from app.models import Document, DocumentPublic, Message
 from app.utils import APIResponse, load_description, get_openai_client
 from app.api.deps import CurrentUser, SessionDep, CurrentUserOrgProject
 from app.core.cloud import AmazonCloudStorage
@@ -29,27 +29,27 @@
 @router.get(
     "/list",
     description=load_description("documents/list.md"),
-    response_model=APIResponse[List[Document]],
+    response_model=APIResponse[List[DocumentPublic]],
 )
 def list_docs(
     session: SessionDep,
-    current_user: CurrentUser,
+    current_user: CurrentUserOrgProject,
     skip: int = Query(0, ge=0),
     limit: int = Query(100, gt=0, le=100),
 ):
-    crud = DocumentCrud(session, current_user.id)
+    crud = DocumentCrud(session, current_user.project_id)
     data = crud.read_many(skip, limit)
     return APIResponse.success_response(data)
 
 
 @router.post(
     "/upload",
     description=load_description("documents/upload.md"),
-    response_model=APIResponse[Document],
+    response_model=APIResponse[DocumentPublic],
 )
 async def upload_doc(
     session: SessionDep,
-    current_user: CurrentUser,
+    current_user: CurrentUserOrgProject,
     src: UploadFile = File(...),
     background_tasks: BackgroundTasks = None,
     target_format: Optional[str] = Form(None),
@@ -62,10 +62,16 @@ async def upload_doc(
         raise HTTPException(status_code=400, detail=str(e))
 
     # Upload the original document first
-    storage = AmazonCloudStorage(current_user)
+    storage = AmazonCloudStorage(current_user.project_id)
     document_id = uuid4()
-    object_store_url = storage.put(src, Path(str(document_id)))
-    crud = DocumentCrud(session, current_user.id)
+    
+    project = get_project_by_id(session=session, project_id=current_user.project_id)
+    if project is None:
+        raise HTTPException(404, "Project not found")
+
+    key = Path(str(project.storage_path), str(document_id))
+    object_store_url = storage.put(src, key)
+    crud = DocumentCrud(session, current_user.project_id)
     document = Document(
         id=document_id,
         fname=src.filename,
@@ -123,10 +129,10 @@ async def upload_doc(
     )
 
 
-@router.get(
+@router.delete(
     "/remove/{doc_id}",
     description=load_description("documents/delete.md"),
-    response_model=APIResponse[Document],
+    response_model=APIResponse[Message],
 )
 def remove_doc(
     session: SessionDep,
@@ -138,18 +144,21 @@ def remove_doc(
     )
 
     a_crud = OpenAIAssistantCrud(client)
-    d_crud = DocumentCrud(session, current_user.id)
+    d_crud = DocumentCrud(session, current_user.project_id)
     c_crud = CollectionCrud(session, current_user.id)
 
     document = d_crud.delete(doc_id)
     data = c_crud.delete(document, a_crud)
-    return APIResponse.success_response(data)
+
+    return APIResponse.success_response(
+        Message(message="Document Deleted Successfully")
+    )
 
 
 @router.delete(
     "/remove/{doc_id}/permanent",
     description=load_description("documents/permanent_delete.md"),
-    response_model=APIResponse[Document],
+    response_model=APIResponse[Message],
 )
 def permanent_delete_doc(
     session: SessionDep,
@@ -161,9 +170,9 @@ def permanent_delete_doc(
     )
 
     a_crud = OpenAIAssistantCrud(client)
-    d_crud = DocumentCrud(session, current_user.id)
+    d_crud = DocumentCrud(session, current_user.project_id)
     c_crud = CollectionCrud(session, current_user.id)
-    storage = AmazonCloudStorage(current_user)
+    storage = AmazonCloudStorage(current_user.project_id)
 
     document = d_crud.read_one(doc_id)
 
@@ -172,19 +181,31 @@ def permanent_delete_doc(
     storage.delete(document.object_store_url)
     d_crud.delete(doc_id)
 
-    return APIResponse.success_response(document)
+    return APIResponse.success_response(
+        Message(message="Document Permanently Deleted Successfully")
+    )
 
 
 @router.get(
     "/info/{doc_id}",
     description=load_description("documents/info.md"),
-    response_model=APIResponse[Document],
+    response_model=APIResponse[DocumentPublic],
 )
 def doc_info(
     session: SessionDep,
-    current_user: CurrentUser,
+    current_user: CurrentUserOrgProject,
     doc_id: UUID = FastPath(description="Document to retrieve"),
+    include_url: bool = Query(
+        False, description="Include a signed URL to access the document"
+    ),
 ):
-    crud = DocumentCrud(session, current_user.id)
-    data = crud.read_one(doc_id)
-    return APIResponse.success_response(data)
+    crud = DocumentCrud(session, current_user.project_id)
+    document = crud.read_one(doc_id)
+
+    doc_schema = DocumentPublic.model_validate(document, from_attributes=True)
+
+    if include_url:
+        storage = AmazonCloudStorage(current_user.project_id)
+        doc_schema.signed_url = storage.get_signed_url(document.object_store_url)
+
+    return APIResponse.success_response(doc_schema)

backend/app/core/cloud/storage.py

@@ -10,7 +10,7 @@
 from botocore.exceptions import ClientError
 from botocore.response import StreamingBody
 
-from app.api.deps import CurrentUser
+from app.models import UserProjectOrg
 from app.core.config import settings
 from app.utils import mask_string
 
@@ -108,8 +108,8 @@ def from_url(cls, url: str):
 
 
 class CloudStorage:
-    def __init__(self, user: CurrentUser):
-        self.user = user
+    def __init__(self, project_id: int):
+        self.project_id = project_id
 
     def put(self, source: UploadFile, basename: str):
         raise NotImplementedError()
@@ -119,12 +119,11 @@ def stream(self, url: str) -> StreamingBody:
 
 
 class AmazonCloudStorage(CloudStorage):
-    def __init__(self, user: CurrentUser):
-        super().__init__(user)
+    def __init__(self, project_id: int):
+        super().__init__(project_id)
         self.aws = AmazonCloudStorageClient()
 
-    def put(self, source: UploadFile, basename: Path) -> SimpleStorageName:
-        key = Path(str(self.user.id), basename)
+    def put(self, source: UploadFile, key: Path) -> SimpleStorageName:
         destination = SimpleStorageName(str(key))
         kwargs = asdict(destination)
 
@@ -138,12 +137,12 @@ def put(self, source: UploadFile, basename: Path) -> SimpleStorageName:
             )
             logger.info(
                 f"[AmazonCloudStorage.put] File uploaded successfully | "
-                f"{{'user_id': '{self.user.id}', 'bucket': '{mask_string(destination.Bucket)}', 'key': '{mask_string(destination.Key)}'}}"
+                f"{{'project_id': '{self.project_id}', 'bucket': '{mask_string(destination.Bucket)}', 'key': '{mask_string(destination.Key)}'}}"
             )
         except ClientError as err:
             logger.error(
                 f"[AmazonCloudStorage.put] AWS upload error | "
-                f"{{'user_id': '{self.user.id}', 'bucket': '{mask_string(destination.Bucket)}', 'key': '{mask_string(destination.Key)}', 'error': '{str(err)}'}}",
+                f"{{'project_id': '{self.project_id}', 'bucket': '{mask_string(destination.Bucket)}', 'key': '{mask_string(destination.Key)}', 'error': '{str(err)}'}}",
                 exc_info=True,
             )
             raise CloudStorageError(f'AWS Error: "{err}"') from err
@@ -157,13 +156,13 @@ def stream(self, url: str) -> StreamingBody:
             body = self.aws.client.get_object(**kwargs).get("Body")
             logger.info(
                 f"[AmazonCloudStorage.stream] File streamed successfully | "
-                f"{{'user_id': '{self.user.id}', 'bucket': '{mask_string(name.Bucket)}', 'key': '{mask_string(name.Key)}'}}"
+                f"{{'project_id': '{self.project_id}', 'bucket': '{mask_string(name.Bucket)}', 'key': '{mask_string(name.Key)}'}}"
             )
             return body
         except ClientError as err:
             logger.error(
                 f"[AmazonCloudStorage.stream] AWS stream error | "
-                f"{{'user_id': '{self.user.id}', 'bucket': '{mask_string(name.Bucket)}', 'key': '{mask_string(name.Key)}', 'error': '{str(err)}'}}",
+                f"{{'project_id': '{self.project_id}', 'bucket': '{mask_string(name.Bucket)}', 'key': '{mask_string(name.Key)}', 'error': '{str(err)}'}}",
                 exc_info=True,
             )
             raise CloudStorageError(f'AWS Error: "{err}" ({url})') from err
@@ -177,13 +176,40 @@ def get_file_size_kb(self, url: str) -> float:
             size_kb = round(size_bytes / 1024, 2)
             logger.info(
                 f"[AmazonCloudStorage.get_file_size_kb] File size retrieved successfully | "
-                f"{{'user_id': '{self.user.id}', 'bucket': '{mask_string(name.Bucket)}', 'key': '{mask_string(name.Key)}', 'size_kb': {size_kb}}}"
+                f"{{'project_id': '{self.project_id}', 'bucket': '{mask_string(name.Bucket)}', 'key': '{mask_string(name.Key)}', 'size_kb': {size_kb}}}"
             )
             return size_kb
         except ClientError as err:
             logger.error(
                 f"[AmazonCloudStorage.get_file_size_kb] AWS head object error | "
-                f"{{'user_id': '{self.user.id}', 'bucket': '{mask_string(name.Bucket)}', 'key': '{mask_string(name.Key)}', 'error': '{str(err)}'}}",
+                f"{{'project_id': '{self.project_id}', 'bucket': '{mask_string(name.Bucket)}', 'key': '{mask_string(name.Key)}', 'error': '{str(err)}'}}",
+                exc_info=True,
+            )
+            raise CloudStorageError(f'AWS Error: "{err}" ({url})') from err
+
+    def get_signed_url(self, url: str, expires_in: int = 3600) -> str:
+        """
+        Generate a signed S3 URL for the given file.
+        :param url: S3 url (e.g., s3://bucket/key)
+        :param expires_in: Expiry time in seconds (default: 1 hour)
+        :return: Signed URL as string
+        """
+        name = SimpleStorageName.from_url(url)
+        try:
+            signed_url = self.aws.client.generate_presigned_url(
+                "get_object",
+                Params={"Bucket": name.Bucket, "Key": name.Key},
+                ExpiresIn=expires_in,
+            )
+            logger.info(
+                f"[AmazonCloudStorage.get_signed_url] Signed URL generated | "
+                f"{{'project_id': '{self.project_id}', 'bucket': '{mask_string(name.Bucket)}', 'key': '{mask_string(name.Key)}'}}"
+            )
+            return signed_url
+        except ClientError as err:
+            logger.error(
+                f"[AmazonCloudStorage.get_signed_url] AWS presign error | "
+                f"{{'project_id': '{self.project_id}', 'bucket': '{mask_string(name.Bucket)}', 'key': '{mask_string(name.Key)}', 'error': '{str(err)}'}}",
                 exc_info=True,
             )
             raise CloudStorageError(f'AWS Error: "{err}" ({url})') from err
@@ -195,12 +221,12 @@ def delete(self, url: str) -> None:
             self.aws.client.delete_object(**kwargs)
             logger.info(
                 f"[AmazonCloudStorage.delete] File deleted successfully | "
-                f"{{'user_id': '{self.user.id}', 'bucket': '{mask_string(name.Bucket)}', 'key': '{mask_string(name.Key)}'}}"
+                f"{{'project_id': '{self.project_id}', 'bucket': '{mask_string(name.Bucket)}', 'key': '{mask_string(name.Key)}'}}"
             )
         except ClientError as err:
             logger.error(
                 f"[AmazonCloudStorage.delete] AWS delete error | "
-                f"{{'user_id': '{self.user.id}', 'bucket': '{mask_string(name.Bucket)}', 'key': '{mask_string(name.Key)}', 'error': '{str(err)}'}}",
+                f"{{'project_id': '{self.project_id}', 'bucket': '{mask_string(name.Bucket)}', 'key': '{mask_string(name.Key)}', 'error': '{str(err)}'}}",
                 exc_info=True,
             )
             raise CloudStorageError(f'AWS Error: "{err}" ({url})') from err