from langchain_community.document_loaders import UnstructuredMarkdownLoader

markdown_path = '/path'

loader = UnstructuredMarkdownLoader(markdown_path)

data = loader.load()

for i in data[0]:   # 查看格式

    print(i)

('page_content', 'layout:     post\ntitle:      利用免费平台加载chatglm\nsubtitle:   飞桨AI studio简介以及chatglm的')

('metadata', {'source': '/Users/klaus_d./Desktop/chatglm.md'})
('type', 'Document')

from langchain_text_splitters import MarkdownHeaderTextSplitter 
# 此工具可以根据文档的标题对文档进行切分
	
markdown_path = 'md = # Foo\n\n ## Bar\n\nHi this is Jim  \nHi this is Joe\n\n ## Baz\n\n Hi this is Molly' 
	
headers_to_split_on = [("#", "Header 1"), ("##", "Header 2"), ("###", "Header 3"),]

markdown_splitter = MarkdownHeaderTextSplitter(headers_to_split_on)
	
md_header_splits = markdown_splitter.split_text(markdown_path)
	
len(md_header_splits) #根据要求被分成了3份
	
for i in md_header_splits[2]:  # 打印分割后文档的第3块输出

    print(i)

('page_content', 'Hi this is Molly')
('metadata', {'Header 2': 'Baz'})
('type', 'Document')

from langchain_text_splitters import CharacterTextSplitter
# 此工具可将文档按字符分割
	
with open('markdown_path') as f:
    state_of_the_union = f.read()

text_splitter = CharacterTextSplitter(
    separator="\n\n",
    chunk_size=300,     # 设置每段分割的长度

    chunk_overlap=0,    # 设置两个分割段落的重叠长度

    length_function=len,
    is_separator_regex=False)
    	
texts = text_splitter.create_documents([state_of_the_union])

for i in texts[:3]:
    print(i.page_content)

len(text_splitter.split_text(state_of_the_union))    # 文档被分割成15段

Created a chunk of size 345, which is longer than the specified 300
Created a chunk of size 345, which is longer than the specified 300
Created a chunk of size 345, which is longer than the specified 300
Created a chunk of size 345, which is longer than the specified 300
227
245
270

from langchain_huggingface import HuggingFaceEmbeddings
	
embeddings_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2", show_progress=True)
	
text_to_embedding = [i.page_content for i in texts]
	
# embed_decuments 接受 list[string] 字符串列表，要对多句话embedding时使用。

embeddings = embeddings_model.embed_documents(text_to_embedding)   

# embed_query 接受一个字符串，对一句话 embedding 时使用。

embedded_query = embeddings_model.embed_query("What was the name mentioned in the conversation?")

HuggingFaceEmbeddings(*, client: Any = None, 
     			model_name: str = 'sentence-transformers/all-mpnet-base-v2', 
 			cache_folder: Optional[str] = None, 
			model_kwargs: Dict[str, Any] = None, 
			encode_kwargs: Dict[str, Any] = None, 
			multi_process: bool = False, 
			show_progress: bool = False) -> None)

from langchain_chroma import Chroma

db = Chroma.from_documents(texts, embeddings_model)

from langchain_community.vectorstores import FAISS

vectorstore = FAISS.from_documents(texts, embeddings_model)

help(FAISS.from_documents)

Help on method from\_documents in module langchain_core.vectorstores:
from_documents(documents: 'List[Document]', embedding: 'Embeddings', **kwargs: 'Any') -> 'VST' method of abc.ABCMeta instance
Return VectorStore initialized from documents and embeddings.

query = "chatglm装在哪里"
	
docs = db.similarity_search(query)

# 默认返回4个结果

Args:
query (str): Query text to search for.
k (int): Number of results to return. Defaults to 4.
filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None.

embedding_vector = embeddings_model.embed_query(query)
		
docs = db.similarity_search_by_vector(embedding_vector)

Args:
embedding (List[float]): Embedding to look up documents similar to.
k (int): Number of Documents to return. Defaults to 4.
filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None.

from langchain_core.prompts import PromptTemplate
	
example_prompt = PromptTemplate.from_template("Question: {question}\n{answer}")
examples = [
	    {
        	"question": "Who lived longer, Muhammad Ali or Alan Turing?",
			"answer": """
	Are follow up questions needed here: Yes.
	Follow up: How old was Muhammad Ali when he died?
	Intermediate answer: Muhammad Ali was 74 years old when he died.
	Follow up: How old was Alan Turing when he died?
	Intermediate answer: Alan Turing was 41 years old when he died.
	So the final answer is: Muhammad Ali
	""",
	    }
	]

Question: Who lived longer, Muhammad Ali or Alan Turing?
Are follow up questions needed here: Yes.
Follow up: How old was Muhammad Ali when he died?
Intermediate answer: Muhammad Ali was 74 years old when he died.
Follow up: How old was Alan Turing when he died?
Intermediate answer: Alan Turing was 41 years old when he died.
So the final answer is: Muhammad Ali

	import requests

	def chat(prompt, history):
	    resp = requests.post(
	        url='http://127.0.0.1:8000',
	        json={'prompt':prompt, 'history':history},
	        headers={'Content-Type': 'application/json;chatsrt=utf-8'}
	    )

	    return resp.json()['response'], resp.json()['history']
	
	history = []
	while True:
	    response, history = chat(input("Question:"), history)
	    print("Anwser:", response)