-
Notifications
You must be signed in to change notification settings - Fork 0
/
ingest.py
30 lines (22 loc) · 962 Bytes
/
ingest.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
from llama_index.readers.json import JSONReader
from llama_index.core import SimpleDirectoryReader, StorageContext
from llama_index.core import VectorStoreIndex, Settings
from llama_index.vector_stores.milvus import MilvusVectorStore
from llama_index.embeddings.fastembed import FastEmbedEmbedding
parser = JSONReader()
file_extractor = {".json": parser} # Add other CSV formats as needed
documents = SimpleDirectoryReader("./data", file_extractor=file_extractor).load_data()
embedding_model = FastEmbedEmbedding(
model_name="mixedbread-ai/mxbai-embed-large-v1", max_length=1024
)
vector_store = MilvusVectorStore(
uri="http://localhost:19530",
collection_name="podcast_data_head",
dim=1024,
overwrite=True,
)
Settings.embed_model = embedding_model
storage_context = StorageContext.from_defaults(vector_store=vector_store)
index = VectorStoreIndex.from_documents(
documents, storage_context=storage_context, show_progress=True
)