(no commit message)
This commit is contained in:
5
LICENSE
5
LICENSE
@@ -1,5 +0,0 @@
|
|||||||
Copyright (C) 2026 by annotateradmin annotateradmin@gmail.com
|
|
||||||
|
|
||||||
Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
|
||||||
52
config.json
Normal file
52
config.json
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
{
|
||||||
|
"model": null,
|
||||||
|
"signature": {
|
||||||
|
"description": "Evaluate and compare the quality of two responses (Response A and Response B) given a specific question.\nDetermine which response better addresses the question by focusing on factual correctness, completeness,\nand adherence to any specific requirements mentioned in the question prompt.\n\nBefore yielding your decision, think step by step and explain your reasoning in the reasoning field.\nBe sure to verbally express your uncertainty in your thought process.\n\nDetailed Instructions:\n\n1. **Understand the Question Context:**\n - Ensure you comprehend the full context and requirements specified by the question or problem statement.\n - Note any domain-specific terminologies or conditions.\n\n2. **Evaluate Each Response:**\n - Check for factual accuracy in the content, calculations, or recommendations provided.\n - Assess the response for completeness\u2014whether it completely addresses all aspects of the question.\n - Verify adherence to the specified question requirements.\n - Consider clarity and structure of the explanation or solution provided.\n\n3. **Decision Making:**\n - Determine which response (A or B) best meets the above criteria.\n - Select the response that is not only correct but also most aligns with the question's specific requirements.\n\n4. **Output Your Conclusion:**\n - Document your reasoning process in the reasoning field.\n - Output \"A>B\" if Response A is better, or \"B>A\" if Response B is better.",
|
||||||
|
"properties": {
|
||||||
|
"question": {
|
||||||
|
"__dspy_field_type": "input",
|
||||||
|
"desc": "The original question or prompt",
|
||||||
|
"prefix": "Question:",
|
||||||
|
"title": "Question",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"response_A": {
|
||||||
|
"__dspy_field_type": "input",
|
||||||
|
"desc": "First response to evaluate",
|
||||||
|
"prefix": "Response A:",
|
||||||
|
"title": "Response A",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"response_B": {
|
||||||
|
"__dspy_field_type": "input",
|
||||||
|
"desc": "Second response to evaluate",
|
||||||
|
"prefix": "Response B:",
|
||||||
|
"title": "Response B",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"reasoning": {
|
||||||
|
"__dspy_field_type": "output",
|
||||||
|
"desc": "Your step by step reasoning for why you chose the better response. With verbally expressed uncertainty.",
|
||||||
|
"prefix": "Reasoning:",
|
||||||
|
"title": "Reasoning",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"label": {
|
||||||
|
"__dspy_field_type": "output",
|
||||||
|
"desc": "Which response is better: 'A>B' or 'B>A'",
|
||||||
|
"prefix": "Label:",
|
||||||
|
"title": "Label",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"question",
|
||||||
|
"response_A",
|
||||||
|
"response_B",
|
||||||
|
"reasoning",
|
||||||
|
"label"
|
||||||
|
],
|
||||||
|
"title": "PreferenceSig",
|
||||||
|
"type": "object"
|
||||||
|
}
|
||||||
|
}
|
||||||
1
probe.json
Normal file
1
probe.json
Normal file
@@ -0,0 +1 @@
|
|||||||
|
{"probe_version":"v1","embedding_dim":5120,"dropout":0.0,"layer_index":16,"num_layers":65,"probe_type":"linear"}
|
||||||
BIN
probe.safetensors
Normal file
BIN
probe.safetensors
Normal file
Binary file not shown.
48
program.json
Normal file
48
program.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"traces": [],
|
||||||
|
"train": [],
|
||||||
|
"demos": [],
|
||||||
|
"signature": {
|
||||||
|
"instructions": "Evaluate and compare the quality of two responses (Response A and Response B) given a specific question.\nDetermine which response better addresses the question by focusing on factual correctness, completeness,\nand adherence to any specific requirements mentioned in the question prompt.\n\nBefore yielding your decision, think step by step and explain your reasoning in the reasoning field.\nBe sure to verbally express your uncertainty in your thought process.\n\nDetailed Instructions:\n\n1. **Understand the Question Context:**\n - Ensure you comprehend the full context and requirements specified by the question or problem statement.\n - Note any domain-specific terminologies or conditions.\n\n2. **Evaluate Each Response:**\n - Check for factual accuracy in the content, calculations, or recommendations provided.\n - Assess the response for completeness\u2014whether it completely addresses all aspects of the question.\n - Verify adherence to the specified question requirements.\n - Consider clarity and structure of the explanation or solution provided.\n\n3. **Decision Making:**\n - Determine which response (A or B) best meets the above criteria.\n - Select the response that is not only correct but also most aligns with the question's specific requirements.\n\n4. **Output Your Conclusion:**\n - Document your reasoning process in the reasoning field.\n - Output \"A>B\" if Response A is better, or \"B>A\" if Response B is better.",
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"prefix": "Question:",
|
||||||
|
"description": "The original question or prompt"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"prefix": "Response A:",
|
||||||
|
"description": "First response to evaluate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"prefix": "Response B:",
|
||||||
|
"description": "Second response to evaluate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"prefix": "Reasoning:",
|
||||||
|
"description": "Your step by step reasoning for why you chose the better response. With verbally expressed uncertainty."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"prefix": "Label:",
|
||||||
|
"description": "Which response is better: 'A>B' or 'B>A'"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"lm": {
|
||||||
|
"model": "together_ai/Qwen/Qwen3-VL-32B-Instruct",
|
||||||
|
"model_type": "chat",
|
||||||
|
"cache": true,
|
||||||
|
"num_retries": 3,
|
||||||
|
"finetuning_model": null,
|
||||||
|
"launch_kwargs": {},
|
||||||
|
"train_kwargs": {},
|
||||||
|
"temperature": null,
|
||||||
|
"max_tokens": null
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"dependency_versions": {
|
||||||
|
"python": "3.13",
|
||||||
|
"dspy": "3.1.3",
|
||||||
|
"cloudpickle": "3.1"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user