AI-Embedding/q3.html at main · Advanced-Topics/AI-Embedding · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
<!DOCTYPE html>
<html lang="en">

<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/water.css@2/out/water.css">
    <title>Question Answering.js</title>
    <style>
        .center {
            display: flex;
            justify-content: center;
            align-items: center;
            height: 100vh;
            flex-direction: column;
        }
    </style>
</head>

<body>
    <h1>Retrieval-based Question-answering</h1>
    <div class="center">
        <ul id="output"></ul>
        <input type="text" id="inputSentence" placeholder="Enter a question" size=60 />
        <button id="searchButton">Answer</button>
    </div>

    <!-- The new module type allows import to be used -->
    <script type="module">

        import { pipeline } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.2';

        // https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.FeatureExtractionPipeline
        const extractor = await pipeline('feature-extraction', 'Xenova/bert-base-uncased', { revision: 'default' });

        // Load questions and answers from the API
        async function loadQuestions() {
            const response = await fetch('https://datasets-server.huggingface.co/rows?dataset=SoorajK1%2Fquestions_and_answers&config=default&split=train&offset=0&length=100');
            const data = await response.json();
            const questionsAndAnswers = data.rows.map(row => ({
                question: row.row.question,
                answer: row.row.answer
            }));
            return questionsAndAnswers;
        }

        // Define an array to store the embeddings of the questions
        const questionEmbeddings = [];
        let questionsAndAnswers = [];

        // Pre-calculate embeddings for all questions
        async function preCalculateEmbeddings() {
            questionsAndAnswers = await loadQuestions();
            for (let i = 0; i < questionsAndAnswers.length; i++) {
                const embed = await extractor(questionsAndAnswers[i].question, { pooling: 'mean', normalize: true });
                questionEmbeddings.push(embed['data']);
            }
            console.log(questionEmbeddings);
        }

        // Call the pre-calculation function
        preCalculateEmbeddings();

        // Function to find the most similar question
        async function findSimilarQuestion() {
            const inputSentence = document.getElementById("inputSentence").value;
            const inputEncoded = await extractor(inputSentence, { pooling: 'mean', normalize: true });
            const inputEmbeddings = inputEncoded['data'];

            let maxSimilarity = -Infinity;
            let mostSimilarQuestion = '';
            let mostSimilarAnswer = '';
            let mostSimilarIndex = -1;

            for (let i = 0; i < questionEmbeddings.length; i++) {
                const similarity = cosineSimilarity(inputEmbeddings, questionEmbeddings[i]);
                if (similarity > maxSimilarity) {
                    maxSimilarity = similarity;
                    mostSimilarQuestion = questionsAndAnswers[i].question;
                    mostSimilarAnswer = questionsAndAnswers[i].answer;
                    mostSimilarIndex = i;
                }
            }

            const output = document.getElementById("output");
            output.innerHTML = "";
            if (mostSimilarQuestion) {
                const listItem = document.createElement("li");
                listItem.innerHTML = `<b>Most simmilar question => </b>${mostSimilarQuestion} (Similarity:<b> ${maxSimilarity.toFixed(2)}</b>)<br><b>Answer =></b>  ${mostSimilarAnswer}`;
                output.appendChild(listItem);
            }
        }

        // Function to calculate cosine similarity
        function cosineSimilarity(vecA, vecB) {
            const dotProduct = vecA.reduce((sum, a, idx) => sum + a * vecB[idx], 0);
            const magnitudeA = Math.sqrt(vecA.reduce((sum, a) => sum + a * a, 0));
            const magnitudeB = Math.sqrt(vecB.reduce((sum, b) => sum + b * b, 0));
            return dotProduct / (magnitudeA * magnitudeB);
        }

        document.getElementById("searchButton").addEventListener("click", findSimilarQuestion);

    </script>
</body>

</html>