Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Multimodal parser uses model gateway, replaced logger.error with logger.exception in try-catch #258

Merged
merged 5 commits into from
Jul 1, 2024
Merged

Multimodal parser uses model gateway, replaced logger.error with logger.exception in try-catch #258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bc68771
Multimodal parser uses, replaced logger.error with logger.exception i…
S1LV3RJ1NX Jun 27, 2024
4449748
Serialized mm parser exception
S1LV3RJ1NX Jun 27, 2024
3ad1309
Merge branch 'main' into ps_multimodal_parser_refactor
S1LV3RJ1NX Jul 1, 2024
f452c53
Resolved the comments
S1LV3RJ1NX Jul 1, 2024
b104d68
Merge branch 'ps_multimodal_parser_refactor' of https://github.com/tr…
S1LV3RJ1NX Jul 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions backend/indexer/indexer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ async def ingest_data_points(
f"Processing done! Deleting file {loaded_data_point.local_filepath}"
)
except Exception as e:
logger.error(
logger.exception(
f"Failed to delete file {loaded_data_point.local_filepath} after processing. Error: {e}"
)
# delete the local_filepath from the loaded_data_point object
Expand All @@ -280,7 +280,7 @@ async def ingest_data_points(
f"Processing done! Deleting file {loaded_data_point.local_metadata_file_path}"
)
except Exception as e:
logger.error(
logger.exception(
f"Failed to delete file {loaded_data_point.local_metadata_file_path} after processing. Error: {e}"
)

Expand Down
58 changes: 29 additions & 29 deletions backend/modules/metadata_store/prismastore.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ async def aconnect(cls, **kwargs):
logger.info(f"Connected to Prisma....")
return cls(db=db, **kwargs)
except Exception as e:
logger.error(f"Failed to connect to Prisma: {e}")
logger.exception(f"Failed to connect to Prisma: {e}")
raise HTTPException(status_code=500, detail="Failed to connect to Prisma")

######
Expand All @@ -50,7 +50,7 @@ async def acreate_collection(self, collection: CreateCollection) -> Collection:
try:
existing_collection = await self.aget_collection_by_name(collection.name)
except Exception as e:
logger.error(f"Error: {e}")
logger.exception(f"Error: {e}")
raise HTTPException(status_code=500, detail=e)

if existing_collection:
Expand All @@ -69,7 +69,7 @@ async def acreate_collection(self, collection: CreateCollection) -> Collection:
collection = await self.db.collection.create(data=collection_data)
return collection
except Exception as e:
logger.error(f"Error: {e}")
logger.exception(f"Error: {e}")
raise HTTPException(status_code=500, detail=f"Error: {e}")

async def aget_collection_by_name(
Expand All @@ -83,7 +83,7 @@ async def aget_collection_by_name(
return collection
return None
except Exception as e:
logger.error(f"Failed to get collection by name: {e}")
logger.exception(f"Failed to get collection by name: {e}")
raise HTTPException(
status_code=500, detail="Failed to get collection by name"
)
Expand All @@ -93,15 +93,15 @@ async def aget_collections(self) -> List[Collection]:
collections = await self.db.collection.find_many()
return collections
except Exception as e:
logger.error(f"Failed to get collections: {e}")
logger.exception(f"Failed to get collections: {e}")
raise HTTPException(status_code=500, detail="Failed to get collections")

async def alist_collections(self) -> List[str]:
try:
collections = await self.aget_collections()
return [collection.name for collection in collections]
except Exception as e:
logger.error(f"Failed to list collections: {e}")
logger.exception(f"Failed to list collections: {e}")
raise HTTPException(status_code=500, detail="Failed to list collections")

async def adelete_collection(self, collection_name: str, include_runs=False):
Expand All @@ -110,7 +110,7 @@ async def adelete_collection(self, collection_name: str, include_runs=False):
if not collection:
logger.debug(f"Collection with name {collection_name} does not exist")
except Exception as e:
logger.debug(e)
logger.exception(e)

try:
await self.db.collection.delete(where={"name": collection_name})
Expand All @@ -120,9 +120,9 @@ async def adelete_collection(self, collection_name: str, include_runs=False):
where={"collection_name": collection_name}
)
except Exception as e:
logger.error(f"Failed to delete data ingestion runs: {e}")
logger.exception(f"Failed to delete data ingestion runs: {e}")
except Exception as e:
logger.error(f"Failed to delete collection: {e}")
logger.exception(f"Failed to delete collection: {e}")
raise HTTPException(status_code=500, detail="Failed to delete collection")

######
Expand All @@ -132,11 +132,11 @@ async def acreate_data_source(self, data_source: CreateDataSource) -> DataSource
try:
existing_data_source = await self.aget_data_source_from_fqn(data_source.fqn)
except Exception as e:
logger.error(f"Error: {e}")
logger.exception(f"Error: {e}")
raise HTTPException(status_code=500, detail=f"Error: {e}")

if existing_data_source:
logger.error(f"Data source with fqn {data_source.fqn} already exists")
logger.exception(f"Data source with fqn {data_source.fqn} already exists")
raise HTTPException(
status_code=400,
detail=f"Data source with fqn {data_source.fqn} already exists",
Expand All @@ -149,7 +149,7 @@ async def acreate_data_source(self, data_source: CreateDataSource) -> DataSource
logger.info(f"Created data source: {data_source}")
return data_source
except Exception as e:
logger.error(f"Failed to create data source: {e}")
logger.exception(f"Failed to create data source: {e}")
raise HTTPException(status_code=500, detail=f"Error: {e}")

async def aget_data_source_from_fqn(self, fqn: str) -> DataSource | None:
Expand All @@ -159,15 +159,15 @@ async def aget_data_source_from_fqn(self, fqn: str) -> DataSource | None:
return data_source
return None
except Exception as e:
logger.error(f"Error: {e}")
logger.exception(f"Error: {e}")
raise HTTPException(status_code=500, detail=f"Error: {e}")

async def aget_data_sources(self) -> List[DataSource]:
try:
data_sources = await self.db.datasource.find_many()
return data_sources
except Exception as e:
logger.error(f"Error: {e}")
logger.exception(f"Error: {e}")
raise HTTPException(status_code=500, detail=f"Error: {e}")

async def aassociate_data_source_with_collection(
Expand All @@ -178,7 +178,7 @@ async def aassociate_data_source_with_collection(
try:
existing_collection = await self.aget_collection_by_name(collection_name)
except Exception as e:
logger.error(f"Error: {e}")
logger.exception(f"Error: {e}")
raise HTTPException(status_code=500, detail=f"Error: {e}")

if not existing_collection:
Expand All @@ -193,7 +193,7 @@ async def aassociate_data_source_with_collection(
data_source_association.data_source_fqn
)
except Exception as e:
logger.error(f"Error: {e}")
logger.exception(f"Error: {e}")
raise HTTPException(status_code=500, detail=f"Error: {e}")

if not data_source:
Expand Down Expand Up @@ -244,7 +244,7 @@ async def aassociate_data_source_with_collection(
return updated_collection

except Exception as e:
logger.error(f"Error: {e}")
logger.exception(f"Error: {e}")
raise HTTPException(
status_code=500,
detail=f"Error: {e}",
Expand All @@ -256,7 +256,7 @@ async def aunassociate_data_source_with_collection(
try:
collection = await self.aget_collection_by_name(collection_name)
except Exception as e:
logger.error(f"Error: {e}")
logger.exception(f"Error: {e}")
raise HTTPException(status_code=500, detail=f"Error: {e}")

if not collection:
Expand All @@ -269,7 +269,7 @@ async def aunassociate_data_source_with_collection(
try:
data_source = await self.aget_data_source_from_fqn(data_source_fqn)
except Exception as e:
logger.error(f"Error: {e}")
logger.exception(f"Error: {e}")
raise HTTPException(status_code=500, detail=f"Error: {e}")

if not data_source:
Expand Down Expand Up @@ -305,7 +305,7 @@ async def aunassociate_data_source_with_collection(
)
return updated_collection
except Exception as e:
logger.error(f"Failed to unassociate data source with collection: {e}")
logger.exception(f"Failed to unassociate data source with collection: {e}")
raise HTTPException(
status_code=500,
detail="Failed to unassociate data source with collection",
Expand All @@ -318,7 +318,7 @@ async def alist_data_sources(
data_sources = await self.aget_data_sources()
return [data_source.dict() for data_source in data_sources]
except Exception as e:
logger.error(f"Failed to list data sources: {e}")
logger.exception(f"Failed to list data sources: {e}")
raise HTTPException(status_code=500, detail="Failed to list data sources")

async def adelete_data_source(self, data_source_fqn: str):
Expand All @@ -332,7 +332,7 @@ async def adelete_data_source(self, data_source_fqn: str):
try:
data_source = await self.aget_data_source_from_fqn(data_source_fqn)
except Exception as e:
logger.error(f"Error: {e}")
logger.exception(f"Error: {e}")
raise HTTPException(status_code=500, detail=f"Error: {e}")

if not data_source:
Expand All @@ -346,7 +346,7 @@ async def adelete_data_source(self, data_source_fqn: str):
try:
collections = await self.aget_collections()
except Exception as e:
logger.error(f"Error: {e}")
logger.exception(f"Error: {e}")
raise HTTPException(status_code=500, detail=f"Error: {e}")

for collection in collections:
Expand Down Expand Up @@ -382,7 +382,7 @@ async def adelete_data_source(self, data_source_fqn: str):
logger.error(f"Folder does not exist: {folder_path}")

except Exception as e:
logger.error(f"Failed to delete data source: {e}")
logger.exception(f"Failed to delete data source: {e}")
raise HTTPException(status_code=500, detail="Failed to delete data source")

######
Expand Down Expand Up @@ -414,7 +414,7 @@ async def acreate_data_ingestion_run(
data_ingestion_run = await self.db.ingestionruns.create(data=run_data)
return DataIngestionRun(**data_ingestion_run.dict())
except Exception as e:
logger.error(f"Failed to create data ingestion run: {e}")
logger.exception(f"Failed to create data ingestion run: {e}")
raise HTTPException(
status_code=500, detail=f"Failed to create data ingestion run: {e}"
)
Expand All @@ -431,7 +431,7 @@ async def aget_data_ingestion_run(
return DataIngestionRun(**data_ingestion_run.dict())
return None
except Exception as e:
logger.error(f"Failed to get data ingestion run: {e}")
logger.exception(f"Failed to get data ingestion run: {e}")
raise HTTPException(status_code=500, detail=f"{e}")

async def aget_data_ingestion_runs(
Expand All @@ -444,7 +444,7 @@ async def aget_data_ingestion_runs(
)
return data_ingestion_runs
except Exception as e:
logger.error(f"Failed to get data ingestion runs: {e}")
logger.exception(f"Failed to get data ingestion runs: {e}")
raise HTTPException(status_code=500, detail=f"{e}")

async def aupdate_data_ingestion_run_status(
Expand All @@ -457,7 +457,7 @@ async def aupdate_data_ingestion_run_status(
)
return updated_data_ingestion_run
except Exception as e:
logger.error(f"Failed to update data ingestion run status: {e}")
logger.exception(f"Failed to update data ingestion run status: {e}")
raise HTTPException(status_code=500, detail=f"{e}")

async def alog_metrics_for_data_ingestion_run(
Expand All @@ -478,7 +478,7 @@ async def alog_errors_for_data_ingestion_run(
data={"errors": json.dumps(errors)},
)
except Exception as e:
logger.error(
logger.exception(
f"Failed to log errors data ingestion run {data_ingestion_run_name}: {e}"
)
raise HTTPException(status_code=500, detail=f"{e}")
Expand Down
Loading
Loading