Tool calls auto calling

.github/workflows/run-unit-tests.yaml

@@ -19,8 +19,11 @@ jobs:
         echo OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} >> .env
         echo LAMOOM_API_URI=${{ secrets.LAMOOM_API_URI }} >> .env
         echo LAMOOM_API_TOKEN=${{ secrets.LAMOOM_API_TOKEN }} >> .env
+        echo LAMOOM_CUSTOM_PROVIDERS=${{ secrets.LAMOOM_CUSTOM_PROVIDERS }} >> .env
         echo NEBIUS_API_KEY=${{ secrets.NEBIUS_API_KEY }} >> .env
         echo CUSTOM_API_KEY=${{ secrets.CUSTOM_API_KEY }} >> .env
+        echo GOOGLE_API_KEY=${{ secrets.GOOGLE_API_KEY }} >> .env
+        echo SEARCH_ENGINE_ID=${{ secrets.SEARCH_ENGINE_ID }} >> .env
         cat .env
 
     - name: Install dependencies
@@ -30,15 +33,21 @@ jobs:
     - name: Install Poetry
       run: pip install poetry
 
+    - name: Cache Poetry Dependencies
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/pypoetry
+        key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-poetry-
+
     - name: Install Python
       uses: actions/setup-python@v3
       with:
         python-version: 3.11
-        cache: poetry
 
     - name: Install Python libraries
-      run: poetry install
+      run: poetry install --with dev
 
     - name: Run tests with pytest
-      run: |
-        poetry run make test
+      run: poetry run make test

.gitignore

@@ -10,4 +10,5 @@ dist
 .vscode
 .pytest_cache
 python
-.env.test
+.env.test
+*/logs/

Makefile

@@ -25,11 +25,7 @@ lint:
 	poetry run isort --settings-path pyproject.toml --check-only .
 
 test:
-	poetry run pytest --cache-clear -vv tests \
-		--cov=${PROJECT_FOLDER} \
-		--cov-config=.coveragerc \
-		--cov-fail-under=81 \
-		--cov-report term-missing
+	poetry run pytest --cache-clear -vv tests
 
 .PHONY: format
 format: make-black isort-check flake8 make-mypy

README.md

@@ -124,6 +124,7 @@ Mix models easily, and districute the load across models. The system will automa
 - Gemini
 - OpenAI (w/ Azure OpenAI models)
 - Nebius with (Llama, DeepSeek, Mistral, Mixtral, dolphin, Qwen and others)
+- OpenRouter woth open source models
 - Custom providers
 
 Model string format is the following for Claude, Gemini, OpenAI, Nebius:
@@ -132,16 +133,17 @@ For Azure models format is the following:
 `"azure/{realm}/{model_name}"`
 
 ```python
-response_llm = client.call(agent.id, context, model = "openai/gpt-4o")
+response_llm = client.call(agent.id, context, model = "openai/o4-mini")
 response_llm = client.call(agent.id, context, model = "azure/useast/gpt-4o")
 ```
 
 Custom model string format is the following:
-`"custom/{model_name}"`
-`provider_url` is required
+`"custom/{provider_name}/{model_name}"`
+where provider is provided in the env variable:
+LAMOOM_CUSTOM_PROVIDERS={"provider_name": {"base_url": "https://","key":"key"}}
 
 ```python
-response_llm = client.call(agent.id, context, model = "custom/gpt-4o", provider_url = "your_model_url")
+response_llm = client.call(agent.id, context, model = "custom/provider_name/model_name")
 ```
 
 ### Lamoom Keys
@@ -169,14 +171,14 @@ prompt.add("You're {name}. Say Hello and ask what's their name.", role="system")
 
 # Call AI model with Lamoom
 context = {"name": "John Doe"}
-response = client.call(prompt.id, context, "openai/gpt-4o")
+response = client.call(prompt.id, context, "openai/o4-mini")
 print(response.content)
 ```
 
 ### Creating Tests While Using Prompts
 ```python
 # Call with test_data to automatically generate tests
-response = client.call(prompt.id, context, "openai/gpt-4o", test_data={
+response = client.call(prompt.id, context, "openai/o4-mini", test_data={
     'ideal_answer': "Hello, I'm John Doe. What's your name?", 
     'model_name': "gemini/gemini-1.5-flash"
 })
@@ -202,6 +204,14 @@ client.add_ideal_answer(
 )
 ```
 
+### To Add Search Credentials:
+- Add Search ENgine id from here:
+https://programmablesearchengine.google.com/controlpanel/create
+
+- Get A google Search Key:
+https://developers.google.com/custom-search/v1/introduction/?apix=true
+
+
 ### Monitoring and Management
 - **Test Dashboard**: Review created tests and scores at https://cloud.lamoom.com/tests
 - **Prompt Management**: Update prompts and rerun tests for published or saved versions
@@ -219,4 +229,5 @@ We welcome contributions! Please see our Contribution Guidelines for more inform
 This project is licensed under the Apache2.0 License - see the [LICENSE](LICENSE.txt) file for details.
 
 ## Contact
-For support or contributions, please contact us via GitHub Issues.
+For support or contributions, please contact us via GitHub Issues.
+

docs/evaluate_prompts_quality/evaluate_prompt_quality.py

@@ -9,20 +9,6 @@
 
 lamoom = Lamoom()
 
-gpt4_behaviour = behaviour.AIModelsBehaviour(
-    attempts=[
-        AttemptToCall(
-            ai_model=AzureAIModel(
-                realm='useast',
-                deployment_id="gpt-4o",
-                max_tokens=C_128K,
-                support_functions=True,
-            ),
-            weight=100,
-        ),
-    ]
-)
-
 
 def main():
     for prompt in get_all_prompts():
@@ -41,7 +27,7 @@ def main():
             'prompt_data': prompt_chats,
             'prompt_id': prompt_id,
         }
-        result = lamoom.call(prompt_to_evaluate_prompt.id, context, gpt4_behaviour)
+        result = lamoom.call(prompt_to_evaluate_prompt.id, context, 'azure/useast/o4-mini')
         print(result.content)
 
 if __name__ == '__main__':