Agent xotirasi va bilimi

import chromadb
from chromadb.utils import embedding_functions
 
# Chroma — sodda vektor bazasi
client = chromadb.Client()
 
# Embedding funksiyasi
embedder = embedding_functions.SentenceTransformerEmbeddingFunction(
    model_name="all-MiniLM-L6-v2"
)
 
kolleksiya = client.create_collection(name="hujjatlar", embedding_function=embedder)
 
# Hujjatlarni qo'shamiz — Chroma ularni avtomatik vektorga aylantiradi
kolleksiya.add(
    documents=[
        "Python — sodda sintaksisli dasturlash tili",
        "JavaScript brauzerda ishlaydi",
        "SQL ma'lumotlar bazasi so'rovlari uchun",
    ],
    ids=["1", "2", "3"],
)
 
# Qidiruv
natija = kolleksiya.query(query_texts=["veb-dasturlash tili"], n_results=2)
print(natija["documents"])
# [['JavaScript brauzerda ishlaydi', 'Python — sodda sintaksisli...']]

from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import PyPDFLoader
 
# PDF yuklaymiz
loader = PyPDFLoader("hujjat.pdf")
sahifalar = loader.load()
 
# Bo'laklarga bo'lamiz
bolgich = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200,
    separators=["\n\n", "\n", ". ", " ", ""],   # tartib bo'yicha sinaydi
)
bolaklar = bolgich.split_documents(sahifalar)
 
print(f"{len(sahifalar)} sahifa → {len(bolaklar)} bo'lak")

from langchain_text_splitters import TokenTextSplitter
 
token_bolgich = TokenTextSplitter(chunk_size=256, chunk_overlap=20)
bolaklar = token_bolgich.split_documents(sahifalar)

def semantik_xotira(savol, xotira_bazasi, client):
    # 1. Savoldan kerakli xotira savolini ajratamiz
    ajrat = client.chat.completions.create(
        model="gpt-4-turbo",
        messages=[{
            "role": "user",
            "content": f"Foydalanuvchi so'rovi: '{savol}'. "
                       f"Bu so'rovga javob berish uchun foydalanuvchi haqida "
                       f"qanday ma'lumot kerak? Faqat qidiruv so'rovini yoz.",
        }],
    )
    xotira_sorovi = ajrat.choices[0].message.content
 
    # 2. Shu so'rov bilan xotiradan qidiramiz
    return xotira_bazasi.query(query_texts=[xotira_sorovi], n_results=3)

from sklearn.cluster import KMeans
import numpy as np
 
 
def xotirani_siq(xotiralar, vektorlar, client, klaster_soni=5):
    """O'xshash xotiralarni guruhlab, har guruhni xulosalaydi."""
    # 1. Klasterlash
    kmeans = KMeans(n_clusters=klaster_soni)
    yorliqlar = kmeans.fit_predict(vektorlar)
 
    siqilgan = []
    for k in range(klaster_soni):
        guruh = [xotiralar[i] for i in range(len(xotiralar)) if yorliqlar[i] == k]
        if not guruh:
            continue
 
        # 2. Har guruhni xulosalaymiz
        xulosa = client.chat.completions.create(
            model="gpt-4-turbo",
            messages=[{
                "role": "user",
                "content": f"Quyidagi o'xshash xotiralarni bitta ixcham "
                           f"faktga birlashtir:\n\n" + "\n".join(guruh),
            }],
        )
        siqilgan.append(xulosa.choices[0].message.content)
 
    return siqilgan