Multi Document Retriever not retrieving all information about llama_index HOT 7 OPEN

response = self._pinecone_index.query(
    vector=query_embedding,
    sparse_vector=sparse_vector,
    top_k=query.similarity_top_k,
    include_values=True,
    include_metadata=True,
    namespace=self.namespace,
    filter=filter,
    **kwargs,
)

top_k_nodes = []
top_k_ids = []
top_k_scores = []
for match in response.matches:
    try:
        node = metadata_dict_to_node(match.metadata)
        node.embedding = match.values
    except Exception:
        _logger.debug("Failed to parse Node metadata, fallback to legacy logic.")
        metadata, node_info, relationships = legacy_metadata_dict_to_node(
            match.metadata, text_key=self.text_key
        )
        text = match.metadata[self.text_key]
        id = match.id
        node = TextNode(
            text=text,
            id_=id,
            metadata=metadata,
            start_char_idx=node_info.get("start", None),
            end_char_idx=node_info.get("end", None),
            relationships=relationships,
        )
    top_k_ids.append(match.id)
    top_k_nodes.append(node)
    top_k_scores.append(match.score)

return VectorStoreQueryResult(
    nodes=top_k_nodes, similarities=top_k_scores, ids=top_k_ids
)

from llama_index.core.retrievers import VectorIndexAutoRetriever
from llama_index.core.vector_stores import MetadataInfo, VectorStoreInfo

vector_store_info = VectorStoreInfo(
    content_info="famous books and movies",
    metadata_info=[
        MetadataInfo(
            name="director",
            type="str",
            description=("Name of the director"),
        ),
        MetadataInfo(
            name="theme",
            type="str",
            description=("Theme of the book/movie"),
        ),
        MetadataInfo(
            name="year",
            type="int",
            description=("Year of the book/movie"),
        ),
    ],
)
retriever = VectorIndexAutoRetriever(
    index,
    vector_store_info=vector_store_info,
    empty_query_top_k=10,
    default_empty_query_vector=[0] * 1536,
    verbose=True,
)

from llama_index.core import VectorStoreIndex, SimpleDirectoryReader

documents = SimpleDirectoryReader("data").load_data()
index = VectorStoreIndex.from_documents(documents)
query_engine = index.as_query_engine(similarity_top_k=5)
response = query_engine.query("What did the author do growing up?")
print(response)

from llama_index.core.query_engine.citation_query_engine import CitationQueryEngine
from llama_index.core.base.base_retriever import BaseRetriever
from llama_index.core.prompts import PromptTemplate

# Define your retriever and other necessary components
retriever = BaseRetriever(...)
citation_qa_template = PromptTemplate(...)
citation_refine_template = PromptTemplate(...)

# Initialize the CitationQueryEngine
citation_query_engine = CitationQueryEngine(
    retriever=retriever,
    citation_qa_template=citation_qa_template,
    citation_refine_template=citation_refine_template,
    citation_chunk_size=512,
    citation_chunk_overlap=20,
)

# Perform a query
response = citation_query_engine.query("Tell me about some open issues related to agents")
print(response)

class DocumentSummaryIndexLLMRetriever(BaseRetriever):
    """Document Summary Index LLM Retriever.

    By default, select relevant summaries from index using LLM calls.

    Args:
        index (DocumentSummaryIndex): The index to retrieve from.
        choice_select_prompt (Optional[BasePromptTemplate]): The prompt to use for selecting relevant summaries.
        choice_batch_size (int): The number of summary nodes to send to LLM at a time.
        choice_top_k (int): The number of summary nodes to retrieve.
        format_node_batch_fn (Callable): Function to format a batch of nodes for LLM.
        parse_choice_select_answer_fn (Callable): Function to parse LLM response.
        service_context (ServiceContext): The service context to use.
    """

    def __init__(
        self,
        index: DocumentSummaryIndex,
        choice_select_prompt: Optional[BasePromptTemplate] = None,
        choice_batch_size: int = 10,
        choice_top_k: int = 1,
        format_node_batch_fn: Optional[Callable] = None,
        parse_choice_select_answer_fn: Optional[Callable] = None,
        service_context: Optional[ServiceContext] = None,
        callback_manager: Optional[CallbackManager] = None,
        object_map: Optional[dict] = None,
        verbose: bool = False,
        **kwargs: Any,
    ) -> None:
        self._index = index
        self._choice_select_prompt = (
            choice_select_prompt or DEFAULT_CHOICE_SELECT_PROMPT
        )
        self._choice_batch_size = choice_batch_size
        self._choice_top_k = choice_top_k
        self._format_node_batch_fn = (
            format_node_batch_fn or default_format_node_batch_fn
        )
        self._parse_choice_select_answer_fn = (
            parse_choice_select_answer_fn or default_parse_choice_select_answer_fn
        )
        self._service_context = service_context or index.service_context
        super().__init__(
            callback_manager=callback_manager, object_map=object_map, verbose=verbose
        )

retriever = DocumentSummaryIndexLLMRetriever(
    index=my_index,
    choice_top_k=5  # Increase this value to retrieve more documents
)

response = retriever.query(
    query="List 2 types of vegetables and how can I grow them",
    similarity_top_k=5  # Adjust this value to retrieve more nodes
)

from llama_index.core import SimpleDirectoryReader
from llama_index.packs.dense_x_retrieval import DenseXRetrievalPack

# Load documents
documents = SimpleDirectoryReader("./data").load_data()

# Use the LLM to extract propositions from every document/node
dense_pack = DenseXRetrievalPack(documents)

# Run the pack
response = dense_pack.run("Physical Standards for Letters?")
print(response)

from llama_index.core.base.base_query_engine import BaseQueryEngine
from llama_index.core.bridge.langchain import BaseTool
from llama_index.core.bridge.pydantic import BaseModel, Field

class IndexToolConfig(BaseModel):
    query_engine: BaseQueryEngine
    name: str
    description: str
    tool_kwargs: Dict = Field(default_factory=dict)

class LlamaIndexTool(BaseTool):
    query_engine: BaseQueryEngine
    return_sources: bool = False

    @classmethod
    def from_tool_config(cls, tool_config: IndexToolConfig) -> "LlamaIndexTool":
        return_sources = tool_config.tool_kwargs.pop("return_sources", False)
        return cls(
            query_engine=tool_config.query_engine,
            name=tool_config.name,
            description=tool_config.description,
            return_sources=return_sources,
            **tool_config.tool_kwargs,
        )

    def _run(self, input: str) -> str:
        response = self.query_engine.query(input)
        if self.return_sources:
            return _get_response_with_sources(response)
        return str(response)

    async def _arun(self, input: str) -> str:
        response = await self.query_engine.aquery(input)
        if self.return_sources:
            return _get_response_with_sources(response)
        return str(response)