{"schema_version":"onlylabs.public_signal.v1","title":"Nous Research Repo: NousResearch/hermes-toolperf-evals","description":"Nous Research repo signal with public source context, captured evidence pages, related signals, and category-scoped analysis context.","url":"https://onlylabs.fyi/signals/1324ed16-f626-4694-b6c3-107d9b659b4c","json_url":"https://onlylabs.fyi/signals/1324ed16-f626-4694-b6c3-107d9b659b4c/signal.json","generated_at":"2026-08-28T18:46:29.577Z","evidence_latest_fetched_at":"2026-08-09T07:02:51.604597+00:00","signal_first_seen_at":"2026-08-09T07:01:34.514829+00:00","org":{"slug":"nous","name":"Nous Research","category":"neolab","category_label":"Neolab","dossier_url":"https://onlylabs.fyi/labs/nous","dossier_json_url":"https://onlylabs.fyi/labs/nous/dossier.json"},"related_urls":{"signal":"https://onlylabs.fyi/signals/1324ed16-f626-4694-b6c3-107d9b659b4c","signal_json":"https://onlylabs.fyi/signals/1324ed16-f626-4694-b6c3-107d9b659b4c/signal.json","source":"https://github.com/NousResearch/hermes-toolperf-evals","lab_dossier":"https://onlylabs.fyi/labs/nous","lab_dossier_json":"https://onlylabs.fyi/labs/nous/dossier.json","analysis":"https://onlylabs.fyi/analysis/nous","analysis_json":"https://onlylabs.fyi/analysis/nous/analysis.json","analysis_evidence_json":"https://onlylabs.fyi/analysis/nous/evidence.json","category":"https://onlylabs.fyi/neolabs","category_json":"https://onlylabs.fyi/neolabs.json","category_feed":"https://onlylabs.fyi/neolabs/feed.xml","category_signals_json":"https://onlylabs.fyi/signals.json?category=neolab","topic":null,"topic_signals_json":null,"topic_feed":null,"data_business":null},"answer_pack":{"answer":"Nous Research published NousResearch/hermes-toolperf-evals (Python). This repository signal exposes tooling, eval, infrastructure, or model-adjacent work before it may appear in a launch post. High-signal details: repo NousResearch/hermes-toolperf-evals · language Python · New eval benchmark repo from NousResearch. onlylabs links this event to 1 captured evidence page and 6 related repo signals.","signal_desk":"repos","source_context":{"source_url":"https://github.com/NousResearch/hermes-toolperf-evals","source_host":"github.com","occurred_at":"2026-08-06T16:14:08+00:00","first_seen_at":"2026-08-09T07:01:34.514829+00:00","date_source":"source","context":"Python"},"context_markers":[{"label":"Lab","value":"Nous Research","source":"signal"},{"label":"Signal desk","value":"repos","source":"signal"},{"label":"Source host","value":"github.com","source":"source"},{"label":"Repository","value":"NousResearch/hermes-toolperf-evals","source":"source"},{"label":"Language","value":"Python","source":"source"},{"label":"Stars","value":"5","source":"traction"},{"label":"Notability","value":"New eval benchmark repo from NousResearch","source":"signal"},{"label":"Watch term","value":"Eval methodology","source":"evidence"},{"label":"Watch term","value":"Data pipeline","source":"evidence"},{"label":"Watch term","value":"Agents and tool use","source":"evidence"}],"evidence_coverage":{"target_pages":1,"captured_pages":1,"readable_pages":1,"capture_methods":["plain"],"missing_page_urls":[],"failed_page_urls":[],"blocked_page_urls":[],"page_urls":["https://github.com/NousResearch/hermes-toolperf-evals"],"related_signals":6,"has_source_url":true,"latest_page_fetched_at":"2026-08-09T07:02:51.604597+00:00"},"data_business":{"matches":false,"lanes":[],"matched_terms":[],"score":null,"reason":null},"agent_handoff":{"signal_json":"https://onlylabs.fyi/signals/1324ed16-f626-4694-b6c3-107d9b659b4c/signal.json","dossier_json":"https://onlylabs.fyi/labs/nous/dossier.json","analysis_json":"https://onlylabs.fyi/analysis/nous/analysis.json","analysis_evidence_json":"https://onlylabs.fyi/analysis/nous/evidence.json","topic_signals_json":null,"topic_feed":null,"category_signals_json":"https://onlylabs.fyi/signals.json?category=neolab","data_radar_json":null,"opportunities_json":null},"analysis_playbook":{"objective":"Turn new repository signals into early evidence of tooling, eval, infrastructure, model-adjacent, or product work before it appears in polished launch channels.","evidence_focus":["repo name","owner","description","language","stars","source URL","first seen time","data, eval, infra, safety, and product terms"],"extraction_questions":["What technical area does this repository expose?","Does the repo imply eval, data, infrastructure, agent, or deployment work?","Is the repo new evidence for a lab direction that is not yet in writing or releases?","Which related signals should an analyst inspect next?"],"signal_questions":["What does this new repository reveal before a formal announcement exists?","What technical area does this repository expose?","Does the repo imply eval, data, infrastructure, agent, or deployment work?","Do the 6 related repo signals show a repeated pattern?"],"output_fields":["org","repo","technical_theme","evidence_url"],"data_business_relevance":"Data-business lane extraction is scoped to frontier labs; for this category, interpret the repository as source-grounded category strategy evidence.","required_sources":[{"label":"signal_json","url":"https://onlylabs.fyi/signals/1324ed16-f626-4694-b6c3-107d9b659b4c/signal.json","required":true},{"label":"source","url":"https://github.com/NousResearch/hermes-toolperf-evals","required":true},{"label":"dossier_json","url":"https://onlylabs.fyi/labs/nous/dossier.json","required":true},{"label":"analysis_evidence_json","url":"https://onlylabs.fyi/analysis/nous/evidence.json","required":true},{"label":"topic_signals_json","url":null,"required":false},{"label":"data_radar_json","url":null,"required":false}],"expected_output":["one-paragraph source-grounded interpretation","category-specific implication","confidence and missing evidence","recommended next source to inspect"],"prompt_seed":"Using only the linked onlylabs JSON, captured source context, and cited evidence, analyze Nous Research's repo signal \"NousResearch/hermes-toolperf-evals\" for neolab strategy."},"semantic_triples":[{"subject":"Nous Research","predicate":"published repo","object":"NousResearch/hermes-toolperf-evals","text":"Nous Research published repo NousResearch/hermes-toolperf-evals."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"is classified as","object":"repo signal","text":"NousResearch/hermes-toolperf-evals is classified as repo signal."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"belongs to","object":"repos desk","text":"NousResearch/hermes-toolperf-evals belongs to repos desk."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has context","object":"Python","text":"NousResearch/hermes-toolperf-evals has context Python."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has evidence coverage","object":"1 captured evidence page","text":"NousResearch/hermes-toolperf-evals has evidence coverage 1 captured evidence page."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has captured page count","object":"1","text":"NousResearch/hermes-toolperf-evals has captured page count 1."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has readable page count","object":"1","text":"NousResearch/hermes-toolperf-evals has readable page count 1."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has related signal count","object":"6","text":"NousResearch/hermes-toolperf-evals has related signal count 6."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has analysis playbook objective","object":"Turn new repository signals into early evidence of tooling, eval, infrastructure, model-adjacent, or product work before it appears in polished launch channels.","text":"NousResearch/hermes-toolperf-evals has analysis playbook objective Turn new repository signals into early evidence of tooling, eval, infrastructure, model-adjacent, or product work before it appears in polished launch channels.."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has source host","object":"github.com","text":"NousResearch/hermes-toolperf-evals has source host github.com."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has lab","object":"Nous Research","text":"NousResearch/hermes-toolperf-evals has lab Nous Research."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has signal desk","object":"repos","text":"NousResearch/hermes-toolperf-evals has signal desk repos."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has source host","object":"github.com","text":"NousResearch/hermes-toolperf-evals has source host github.com."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has repository","object":"NousResearch/hermes-toolperf-evals","text":"NousResearch/hermes-toolperf-evals has repository NousResearch/hermes-toolperf-evals."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has language","object":"Python","text":"NousResearch/hermes-toolperf-evals has language Python."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has stars","object":"5","text":"NousResearch/hermes-toolperf-evals has stars 5."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has notability","object":"New eval benchmark repo from NousResearch","text":"NousResearch/hermes-toolperf-evals has notability New eval benchmark repo from NousResearch."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has watch term","object":"Eval methodology","text":"NousResearch/hermes-toolperf-evals has watch term Eval methodology."}]},"intelligence":{"signal_desk":"repos","answer":"Nous Research published NousResearch/hermes-toolperf-evals (Python). This repository signal exposes tooling, eval, infrastructure, or model-adjacent work before it may appear in a launch post. High-signal details: repo NousResearch/hermes-toolperf-evals · language Python · New eval benchmark repo from NousResearch. onlylabs links this event to 1 captured evidence page and 6 related repo signals.","semantic_triples":[{"subject":"Nous Research","predicate":"published repo","object":"NousResearch/hermes-toolperf-evals","text":"Nous Research published repo NousResearch/hermes-toolperf-evals."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"is classified as","object":"repo signal","text":"NousResearch/hermes-toolperf-evals is classified as repo signal."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"belongs to","object":"repos desk","text":"NousResearch/hermes-toolperf-evals belongs to repos desk."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has context","object":"Python","text":"NousResearch/hermes-toolperf-evals has context Python."},{"subject":"NousResearch/hermes-toolperf-evals","predicate":"has evidence coverage","object":"1 captured evidence page","text":"NousResearch/hermes-toolperf-evals has evidence coverage 1 captured evidence page."}]},"signal":{"id":"1324ed16-f626-4694-b6c3-107d9b659b4c","url":"https://onlylabs.fyi/signals/1324ed16-f626-4694-b6c3-107d9b659b4c","json_url":"https://onlylabs.fyi/signals/1324ed16-f626-4694-b6c3-107d9b659b4c/signal.json","source_url":"https://github.com/NousResearch/hermes-toolperf-evals","title":"NousResearch/hermes-toolperf-evals","summary":"Nous Research published a new repository. onlylabs watches repos for tooling, eval, infra, and model-adjacent work.","context":"Python","kind":{"key":"repo_new","label":"Repo"},"org":{"slug":"nous","name":"Nous Research","category":"neolab"},"occurred_at":"2026-08-06T16:14:08+00:00","first_seen_at":"2026-08-09T07:01:34.514829+00:00","date_source":"source","evidence_coverage":{"target_pages":1,"captured_pages":1,"readable_pages":1,"capture_methods":["plain"],"missing_page_urls":[],"failed_page_urls":[],"blocked_page_urls":[],"page_urls":["https://github.com/NousResearch/hermes-toolperf-evals"]},"facets":{"repo":"NousResearch/hermes-toolperf-evals","language":"Python"},"traction":{"github_stars":5,"hn_points":null,"hn_comments":null,"hn_story_id":null,"hf_downloads":null,"hf_likes":null},"data_radar":null},"primary_evidence_page":{"is_primary":true,"source_match":true,"url":"https://github.com/NousResearch/hermes-toolperf-evals","final_url":"https://github.com/NousResearch/hermes-toolperf-evals","title":"NousResearch/hermes-toolperf-evals repository metadata","http_status":200,"content_type":"application/json","capture_method":"plain","fetched_at":"2026-08-09T07:02:51.604597+00:00","bytes":11087,"raw_path":"5b5e3a4b2e15c7a25e25e6747361f6185e1c71e09c6d25406559bfa857bdc96f.json","content_hash":"e037ddbb27b9b4128b2230ed948af6910ded92c3e112838ff27a24533eb5257a","excerpt_chars":1200,"truncated":true,"excerpt":"NousResearch/hermes-toolperf-evals Description: Core-toolset A/B eval harness + benchmark cases behind the 15-PR tool-efficiency tracker (hermes-agent #77056). Built from session-DB mining, NeMo Relay traces, and prod error taxonomy. Language: Python Stars: 0 Forks: 0 Open issues: 0 Created: 2026-08-06T16:14:08Z Pushed: 2026-08-06T17:16:06Z Default branch: main Fork: no Archived: no README: hermes-toolperf-evals The complete eval set behind the hermes-agent core-toolset performance batch — 15 PRs + 1 integration fix, all merged (tracker: [hermes-agent#77056](https://github.com/NousResearch/hermes-agent/issues/77056)). Built from production session-DB mining (~1.5M messages), a schema token audit, and NVIDIA NeMo Relay ATOF traces for ground-truth turn accounting. Layout | dir | what | |---|---| | `abeval/` | The hard A/B eval: 9 error-inducing benchmark cases, baseline-tree vs fixes-tree (arms differ only by `PYTHONPATH`), 2 models × 3 reps, ATOF-scored. This is the harness behind the published before/after numbers. Parameterized — see its README. | | `bench/` | The earlier 11-task baseline battery (8 generic + 3 hint-targeted) that established the waste profile. Preserved as..."},"evidence_pages":[],"related_signals":[{"id":"0d7503da-5bbd-43c9-804b-27b8039941ae","url":"https://onlylabs.fyi/signals/0d7503da-5bbd-43c9-804b-27b8039941ae","source_url":"https://github.com/NousResearch/hermes-plugin-sprites","title":"NousResearch/hermes-plugin-sprites","context":"Python","kind":{"key":"repo_new","label":"Repo"},"org":{"slug":"nous","name":"Nous Research","category":"neolab"},"occurred_at":"2026-08-25T03:06:21+00:00","first_seen_at":"2026-08-28T07:00:44.105772+00:00","date_source":"source"},{"id":"549a0971-7bab-4f77-8e9e-7e637b1daad8","url":"https://onlylabs.fyi/signals/549a0971-7bab-4f77-8e9e-7e637b1daad8","source_url":"https://github.com/NousResearch/Hermes-Bot-Mode","title":"NousResearch/Hermes-Bot-Mode","context":"JavaScript","kind":{"key":"repo_new","label":"Repo"},"org":{"slug":"nous","name":"Nous Research","category":"neolab"},"occurred_at":"2026-08-13T16:40:20+00:00","first_seen_at":"2026-08-14T07:00:44.351967+00:00","date_source":"source"},{"id":"08883fe2-5f85-4a7f-96ed-1ef3f5acecb5","url":"https://onlylabs.fyi/signals/08883fe2-5f85-4a7f-96ed-1ef3f5acecb5","source_url":"https://github.com/NousResearch/hermes-e2e-evidence","title":"NousResearch/hermes-e2e-evidence","context":null,"kind":{"key":"repo_new","label":"Repo"},"org":{"slug":"nous","name":"Nous Research","category":"neolab"},"occurred_at":"2026-07-23T00:05:13+00:00","first_seen_at":"2026-07-23T07:00:45.549414+00:00","date_source":"source"},{"id":"0dbcd6e6-8bd4-49c1-892b-07a09e5f15bf","url":"https://onlylabs.fyi/signals/0dbcd6e6-8bd4-49c1-892b-07a09e5f15bf","source_url":"https://github.com/NousResearch/hermes-agent-ci-infra","title":"NousResearch/hermes-agent-ci-infra","context":"Shell","kind":{"key":"repo_new","label":"Repo"},"org":{"slug":"nous","name":"Nous Research","category":"neolab"},"occurred_at":"2026-07-17T20:52:01+00:00","first_seen_at":"2026-07-21T07:01:43.587157+00:00","date_source":"source"},{"id":"923e7066-58b5-4514-a2e0-5b0afc6abf8b","url":"https://onlylabs.fyi/signals/923e7066-58b5-4514-a2e0-5b0afc6abf8b","source_url":"https://github.com/NousResearch/hermes-telegram-business","title":"NousResearch/hermes-telegram-business","context":"Python","kind":{"key":"repo_new","label":"Repo"},"org":{"slug":"nous","name":"Nous Research","category":"neolab"},"occurred_at":"2026-07-05T21:41:56+00:00","first_seen_at":"2026-07-06T07:01:39.434893+00:00","date_source":"source"},{"id":"8c537d36-674f-4a6d-8078-e61250d0fead","url":"https://onlylabs.fyi/signals/8c537d36-674f-4a6d-8078-e61250d0fead","source_url":"https://github.com/NousResearch/hermes-compression-eval","title":"NousResearch/hermes-compression-eval","context":"Python","kind":{"key":"repo_new","label":"Repo"},"org":{"slug":"nous","name":"Nous Research","category":"neolab"},"occurred_at":"2026-05-16T09:41:37+00:00","first_seen_at":"2026-06-06T01:49:37.982614+00:00","date_source":"source"}]}