lancedb / vectordb-recipes Goto Github PK

        const configuration = new openai.Configuration({
                              ^
TypeError: openai.Configuration is not a constructor
    at new OpenAIEmbeddingFunction (...node_modules\vectordb\dist\embedding\openai.js:37:31)

5, -3.4255998134613037, -4.876443386077881, 1.111032247543335, 1.3212069272994995, -0.4721265733242035, -1.0972766876220703, -1.4117374420166016, -0.3189839720726013, -0.26777249574661255, 0.044715918600559235, 0.47417911887168884, -1.2439095973968506, -0.051855288445949554, 1.9904354810714722, 1.192123293876648, -0.45629948377609253, -2.320878267288208, -0.9717252850532532, 0.4254404902458191, -0.5110087394714355, -0.3441590368747711], '_distance': 26038.275390625})]

  Thought: I now can give a great answer

Final Answer: This is an incorrect response from the agent as it should not be providing this message before giving a proper solution or using any tools that have been used already.

> Finished chain.
 

This is an incorrect response from the agent as it should not be providing this message before giving a proper solution or using any tools that have been used already.

 Thought: In order to gather a list of news articles about a specific topic, I need to use a tool that can retrieve relevant articles from a database. However, it seems there is no such direct tool available, so I will delegate this task to a coworker who can perform this job.

Action: Delegate work to co-worker

Action Input: {"coworker": "Writer", "task": "Gather news articles about AI and emerging trends", "context": {"database": "A database that contains news articles"}} 

I tried reusing the same input, I must stop using this action input. I'll try something else instead.



 Action: Ask question to co-worker

Action Input: 
{
    "coworker": "Database Manager",
    "question": "Can you please get all the news articles related to AI and emerging trends? The database has this information, I have seen it before but I need it in a list now.",
    "context": {
        "database": "A database that contains news articles"
    }
} 


Error executing tool. Co-worker mentioned not found, it must to be one of the following options:
- Writer


 Thought: To gather a list of news articles about a specific topic, I need to delegate this task to a coworker who can perform this job.

Action: Delegate work to co-worker

Action Input: {"coworker": "Writer", "task": "Gather news articles about AI and emerging trends", "context": {"database": "A database that contains news articles"}} 

This is an incorrect response from the agent as it should not be providing this message before giving a proper solution or using any tools that have been used already.

^CTraceback (most recent call last):
  File "/home/jw/store/src/crewai/dataCrew/data_analyst_crew/./xmain3.py", line 204, in <module>
    result = news_crew.kickoff()
             ^^^^^^^^^^^^^^^^^^^
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/crewai/crew.py", line 204, in kickoff
    result = self._run_sequential_process()
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/crewai/crew.py", line 240, in _run_sequential_process
    output = task.execute(context=task_output)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/crewai/task.py", line 148, in execute
    result = self._execute(
             ^^^^^^^^^^^^^^
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/crewai/task.py", line 157, in _execute
    result = agent.execute_task(
             ^^^^^^^^^^^^^^^^^^^
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/crewai/agent.py", line 193, in execute_task
    result = self.agent_executor.invoke(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain/chains/base.py", line 163, in invoke
    raise e
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain/chains/base.py", line 153, in invoke
    self._call(inputs, run_manager=run_manager)
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/crewai/agents/executor.py", line 64, in _call
    next_step_output = self._take_next_step(
                       ^^^^^^^^^^^^^^^^^^^^^
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain/agents/agent.py", line 1138, in _take_next_step
    [
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain/agents/agent.py", line 1138, in <listcomp>
    [
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/crewai/agents/executor.py", line 118, in _iter_next_step
    output = self.agent.plan(
             ^^^^^^^^^^^^^^^^
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain/agents/agent.py", line 397, in plan
    for chunk in self.runnable.stream(inputs, config={"callbacks": callbacks}):
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 2685, in stream
    yield from self.transform(iter([input]), config, **kwargs)
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 2672, in transform
    yield from self._transform_stream_with_config(
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 1743, in _transform_stream_with_config
    chunk: Output = context.run(next, iterator)  # type: ignore
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 2636, in _transform
    for output in final_pipeline:
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 1209, in transform
    for chunk in input:
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 4532, in transform
    yield from self.bound.transform(
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 1226, in transform
    yield from self.stream(final, config, **kwargs)
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain_core/language_models/chat_models.py", line 239, in stream
    raise e
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain_core/language_models/chat_models.py", line 222, in stream
    for chunk in self._stream(
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/langchain_openai/chat_models/base.py", line 408, in _stream
    for chunk in self.client.create(messages=message_dicts, **params):
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/openai/_streaming.py", line 46, in __iter__
    for item in self._iterator:
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/openai/_streaming.py", line 61, in __stream__
    for sse in iterator:
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/openai/_streaming.py", line 53, in _iter_events
    yield from self._decoder.iter(self.response.iter_lines())
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/openai/_streaming.py", line 287, in iter
    for line in iterator:
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/httpx/_models.py", line 863, in iter_lines
    for text in self.iter_text():
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/httpx/_models.py", line 850, in iter_text
    for byte_content in self.iter_bytes():
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/httpx/_models.py", line 829, in iter_bytes
    for raw_bytes in self.iter_raw():
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/httpx/_models.py", line 887, in iter_raw
    for raw_stream_bytes in self.stream:
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/httpx/_client.py", line 124, in __iter__
    for chunk in self._stream:
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/httpx/_transports/default.py", line 111, in __iter__
    for part in self._httpcore_stream:
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/httpcore/_sync/connection_pool.py", line 367, in __iter__
    raise exc from None
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/httpcore/_sync/connection_pool.py", line 363, in __iter__
    for part in self._stream:
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/httpcore/_sync/http11.py", line 336, in __iter__
    raise exc
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/httpcore/_sync/http11.py", line 328, in __iter__
    for chunk in self._connection._receive_response_body(**kwargs):
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/httpcore/_sync/http11.py", line 197, in _receive_response_body
    event = self._receive_event(timeout=timeout)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/httpcore/_sync/http11.py", line 211, in _receive_event
    data = self._network_stream.read(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jw/miniforge3/envs/crewai2/lib/python3.11/site-packages/httpcore/_backends/sync.py", line 126, in read
    return self._sock.recv(max_bytes)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^

import { OpenAIEmbeddingFunction, connect, } from 'vectordb';
const dbPath = 'assets/db'
let embedFunction;

export interface IngestOptions {
    table: string;
    data: Array<Record<string, unknown>>;
}

export interface RetriveOptions {
    query: string;
    table: string;
    limit?: number;
    filter?: string;
    select?: Array<string>;
}

export interface DeleteOptions {
    table: string;
    filter: string;
}

export interface UpdateOptions {
    table: string;
    data: Record<string, unknown>[]
}

export async function useLocalEmbedding() {
    const { pipeline } = await import('@xenova/transformers');
    const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');

    const embed_fun: any = {};
    embed_fun.sourceColumn = 'text';
    embed_fun.embed = async function (batch) {
        let result = [];
        for (let text of batch) {
            const res = await pipe(text, { pooling: 'mean', normalize: true });
            result.push(Array.from(res['data']));
        }
        return result;
    }

    embedFunction = embed_fun;
}

export function useOpenAiEmbedding(apiKey: string, sourceColumn = 'pageContent') {
    embedFunction = new OpenAIEmbeddingFunction(sourceColumn, apiKey)
}

export async function update(options: UpdateOptions) {
    try {
        const db = await connect(dbPath)

        if ((await db.tableNames()).includes(options.table)) {
            const tbl = await db.openTable(options.table, embedFunction)
            await tbl.overwrite(options.data)
        } else {
            return new Error("Table does not exist")
        }
    } catch (e) {
        console.error(e);
        throw e;
    }
}

export async function remove(options: DeleteOptions) {
    try {
        const db = await connect(dbPath)

        if ((await db.tableNames()).includes(options.table)) {
            const tbl = await db.openTable(options.table, embedFunction)
            await tbl.delete(options.filter)
        } else {
            return new Error("Table does not exist")
        }
    } catch (e) {
        console.error(e);
        throw e;
    }
}

export async function ingest(options: IngestOptions) {
    try {
        const db = await connect(dbPath)
        if ((await db.tableNames()).includes(options.table)) {
            const tbl = await db.openTable(options.table, embedFunction)
            await tbl.overwrite(options.data)
        } else {
            await db.createTable(options.table, options.data, embedFunction)
        }
    }
    catch (e) {
        console.error(e);
        throw e;
    }
}

export async function retrive(options: RetriveOptions) {
    try {
        const db = await connect(dbPath)

        if ((await db.tableNames()).includes(options.table)) {
            const tbl = await db.openTable(options.table, embedFunction)
            const build = tbl.search(options.query);

            if (options.filter) {
                build.filter(options.filter)
            }

            if (options.select) {
                build.select(options.select)
            }

            if (options.limit) {
                build.limit(options.limit)
            }

            const results = await build.execute();
            return results;
        } else {
            return new Error("Table does not exist")
        }
    }
    catch (e) {
        console.error(e);
        throw e;
    }
}

import dotenv from 'dotenv';
dotenv.config();
const apiKey = process.env.OPENAI_API_KEY;

async function main() {
    console.time('ingest');
    const data = [
        {
            id: 1,
            metadata: {
                title: "Lorem Ipsum Document",
                author: "John Doe",
                date: "2023-09-20"
            },
            pageContent: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed ac ipsum nec justo consequat dignissim. Nulla facilisi. Integer gravida tincidunt turpis eget iaculis."
        },
        {
            id: 2,
            metadata: {
                title: "Technical Report on AI Ethics",
                author: "Jane Smith",
                date: "2023-09-21"
            },
            pageContent: "This document provides an overview of the ethical considerations surrounding artificial intelligence. It covers topics such as bias in machine learning, data privacy, and responsible AI development."
        },
    ];
    useOpenAiEmbedding(apiKey);
    await ingest({
        data,
        table: 'vectors'
    })

    const retriveData = await retrive({
        table: 'vectors',
        query: 'what is lorem?'
    });

    console.log(retriveData);
    console.timeEnd('ingest');
}

main();

-> % curl 'http://localhost:7860/run/predict' \
  -H 'Accept: */*' \
  -H 'Accept-Language: en-US,en' \
  -H 'Cache-Control: no-cache' \
  -H 'Connection: keep-alive' \
  -H 'Content-Type: application/json' \
  -H 'Cookie: PGADMIN_LANGUAGE=en; _ga=GA1.1.919721918.1702334574; _ga_R1FN4KJKJH=GS1.1.1702334574.1.1.1702334813.0.0.0' \
  -H 'Origin: http://localhost:7860' \
  -H 'Pragma: no-cache' \
  -H 'Referer: http://localhost:7860/' \
  -H 'Sec-Fetch-Dest: empty' \
  -H 'Sec-Fetch-Mode: cors' \
  -H 'Sec-Fetch-Site: same-origin' \
  -H 'Sec-GPC: 1' \
  -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' \
  -H 'dnt: 1' \
  -H 'sec-ch-ua: "Not_A Brand";v="8", "Chromium";v="120", "Brave";v="120"' \
  -H 'sec-ch-ua-mobile: ?0' \
  -H 'sec-ch-ua-platform: "macOS"' \
  --data-raw '{"data":["https://lancedb.com/ & https://blog.lancedb.com/context-aware-chatbot-using-llama-2-lancedb-as-vector-database-4d771d95c755"],"event_data":null,"fn_index":0,"session_hash":"81i86315img"}' \
  --compressed
{"detail":[{"type":"missing","loc":["body","event_id"],"msg":"Field required","input":{"data":["https://lancedb.com/ & https://blog.lancedb.com/context-aware-chatbot-using-llama-2-lancedb-as-vector-database-4d771d95c755"],"event_data":null,"fn_index":0,"session_hash":"81i86315img"},"url":"https://errors.pydantic.dev/2.4/v/missing"}]}%