KeyArgo
/
private-gpt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Better logs during ingestion

This commit is contained in:
Iván Martínez 2023-05-20 12:11:21 +02:00
parent 4a0e0d2e70
commit 80b9b1d03e
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ingest.py
View File

@ -149,15 +149,19 @@ def main():
db = Chroma(persist_directory=persist_directory, embedding_function=embeddings, client_settings=CHROMA_SETTINGS) db = Chroma(persist_directory=persist_directory, embedding_function=embeddings, client_settings=CHROMA_SETTINGS)
collection = db.get() collection = db.get()
texts = process_documents([metadata['source'] for metadata in collection['metadatas']]) texts = process_documents([metadata['source'] for metadata in collection['metadatas']])
print(f"Creating embeddings. May take some minutes...")
db.add_documents(texts) db.add_documents(texts)
else: else:
# Create and store locally vectorstore # Create and store locally vectorstore
print("Creating new vectorstore") print("Creating new vectorstore")
texts = process_documents() texts = process_documents()
print(f"Creating embeddings. May take some minutes...")
db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory, client_settings=CHROMA_SETTINGS) db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory, client_settings=CHROMA_SETTINGS)
db.persist() db.persist()
db = None db = None
print(f"Ingestion complete! You can now run privateGPT.py to query your documents")
if __name__ == "__main__": if __name__ == "__main__":
main() main()