From e46e976aef92f08088ed605c8ef155b657f88d02 Mon Sep 17 00:00:00 2001 From: Kshitij Chauhan Date: Fri, 6 Nov 2026 20:24:22 +0000 Subject: [PATCH 1/5] Separate content labels and actions --- crates/ingest-consumer/src/main.rs | 9 +- crates/moderation/src/fast_path.rs | 30 +++--- crates/moderation/src/lib.rs | 146 ++++------------------------- crates/moderation/src/questions.rs | 18 ++-- crates/moderation/src/verdict.rs | 137 +++++++++++++++++++++++++++ 5 files changed, 182 insertions(+), 158 deletions(-) create mode 100644 crates/moderation/src/verdict.rs diff --git a/crates/ingest-consumer/src/main.rs b/crates/ingest-consumer/src/main.rs index eb3e44e..aac139c 100644 --- a/crates/ingest-consumer/src/main.rs +++ b/crates/ingest-consumer/src/main.rs @@ -424,7 +424,7 @@ async fn consume_labels( // Only the version of a record that is currently indexed is // moderated. Older versions are skipped, since a later message // covers the newer one. - let inference = match collection.as_deref() { + let verdict = match collection.as_deref() { Some(RouteRecord::NSID) => { let Some(route) = fetch_route_moderation_view(pool, &uri) .await? @@ -481,7 +481,8 @@ async fn consume_labels( } }; - if inference.labels.is_empty() { + let labels = verdict.vals(); + if labels.is_empty() { info!(uri, cid, "no labels applied"); return Ok(()); } @@ -489,9 +490,9 @@ async fn consume_labels( let labels = NewLabels { uri, cid, - vals: inference.labels.iter().map(ToString::to_string).collect(), + vals: labels.iter().map(ToString::to_string).collect(), cts: Timestamp::now(), - source: inference.source.as_str().to_owned(), + source: verdict.source.as_str().to_owned(), }; database::labels::insert_labels(pool, &labels).await?; info!(uri = labels.uri, cid = labels.cid, vals = ?labels.vals, "labels applied"); diff --git a/crates/moderation/src/fast_path.rs b/crates/moderation/src/fast_path.rs index 08e98ce..266ff1e 100644 --- a/crates/moderation/src/fast_path.rs +++ b/crates/moderation/src/fast_path.rs @@ -1,30 +1,30 @@ use rustrict::{Censor, Type}; use serde_json::Value; -use crate::{LabelVal, with_action}; +use crate::verdict::ContentLabel; /// Only terms rustrict rates as severe may bypass the decisions API. const MIN_SEVERITY: Type = Type::SEVERE; /// A function that returns labels for text, or `None` if the result is /// ambiguous and must be decided by the model. -type LabelCheck = fn(&str) -> Option>; +type LabelCheck = fn(&str) -> Option>; /// Map a rustrict analysis to labels without involving any real text. /// /// Severe `mean` matches are ambiguous and return `None`: rustrict uses `mean` /// for both ordinary insults and violent language, which need different labels. -fn labels_for_analysis(analysis: Type) -> Option> { +fn labels_for_analysis(analysis: Type) -> Option> { if analysis.is(Type::MEAN & MIN_SEVERITY) { return None; } let mut labels = Vec::new(); if analysis.is((Type::PROFANE | Type::OFFENSIVE) & MIN_SEVERITY) { - labels.push(LabelVal::Rude); + labels.push(ContentLabel::Rude); } if analysis.is(Type::SEXUAL & MIN_SEVERITY) { - labels.push(LabelVal::Porn); + labels.push(ContentLabel::Porn); } Some(labels) @@ -34,7 +34,7 @@ fn labels_for_analysis(analysis: Type) -> Option> { /// /// Returns `None` when the text contains an ambiguous severe `mean` match. #[must_use] -fn text_labels(text: &str) -> Option> { +fn text_labels(text: &str) -> Option> { if text.trim().is_empty() { return Some(Vec::new()); } @@ -53,17 +53,17 @@ fn text_labels(text: &str) -> Option> { /// the whole record to skip the decisions API, unless any field has an /// ambiguous severe `mean` match. #[must_use] -pub(crate) fn record_labels(record: &Value) -> Vec { +pub(crate) fn record_labels(record: &Value) -> Vec { let mut labels = Vec::new(); if collect(record, &mut labels, text_labels) { return Vec::new(); } - with_action(labels) + labels } /// Collect labels, returning `true` if any value is ambiguous. -fn collect(value: &Value, labels: &mut Vec, check: LabelCheck) -> bool { +fn collect(value: &Value, labels: &mut Vec, check: LabelCheck) -> bool { match value { Value::String(text) => match check(text) { Some(found) => { @@ -92,7 +92,7 @@ mod tests { fn severe_profanity_is_rude() { assert_eq!( labels_for_analysis(Type::PROFANE & Type::SEVERE), - Some(vec![LabelVal::Rude]) + Some(vec![ContentLabel::Rude]) ); } @@ -100,7 +100,7 @@ mod tests { fn severe_offensive_is_rude() { assert_eq!( labels_for_analysis(Type::OFFENSIVE & Type::SEVERE), - Some(vec![LabelVal::Rude]) + Some(vec![ContentLabel::Rude]) ); } @@ -108,7 +108,7 @@ mod tests { fn severe_sexual_is_porn() { assert_eq!( labels_for_analysis(Type::SEXUAL & Type::SEVERE), - Some(vec![LabelVal::Porn]) + Some(vec![ContentLabel::Porn]) ); } @@ -141,9 +141,9 @@ mod tests { /// A stand-in for `text_labels` so recursion can be tested without /// embedding real moderation terms in the source. - fn fake_check(text: &str) -> Option> { + fn fake_check(text: &str) -> Option> { match text { - "flagged" => Some(vec![LabelVal::Rude]), + "flagged" => Some(vec![ContentLabel::Rude]), "unclear" => None, _ => Some(Vec::new()), } @@ -158,7 +158,7 @@ mod tests { &mut labels, fake_check )); - assert_eq!(labels, vec![LabelVal::Rude]); + assert_eq!(labels, vec![ContentLabel::Rude]); } #[test] diff --git a/crates/moderation/src/lib.rs b/crates/moderation/src/lib.rs index e21652e..e8a2a32 100644 --- a/crates/moderation/src/lib.rs +++ b/crates/moderation/src/lib.rs @@ -1,96 +1,17 @@ use anyhow::{Result, anyhow}; use serde_json::{Map, Value}; -use std::{collections::HashMap, fmt::Display, hash::BuildHasher, time::Duration}; +use std::{collections::HashMap, time::Duration}; use tracing::{debug, instrument}; -use crate::questions::{JUDGMENTS, TRUE_THRESHOLD}; +use crate::{ + questions::{JUDGMENTS, TRUE_THRESHOLD}, + verdict::{ContentLabel, Source, Verdict}, +}; mod fast_path; mod questions; mod typesafe; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum LabelVal { - // ATProto global values - Hide, - Warn, - Porn, - Sexual, - Nudity, - GraphicMedia, - // ATTrails values - Spam, - Rude, - Threat, -} - -impl LabelVal { - #[must_use] - pub fn as_str(self) -> &'static str { - match self { - LabelVal::Hide => "!hide", - LabelVal::Warn => "!warn", - LabelVal::Porn => "porn", - LabelVal::Sexual => "sexual", - LabelVal::Nudity => "nudity", - LabelVal::GraphicMedia => "graphic-media", - LabelVal::Spam => "spam", - LabelVal::Rude => "rude", - LabelVal::Threat => "threat", - } - } -} - -impl Display for LabelVal { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} - -/// Where the labels for a record came from. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum LabelSource { - /// Rustrict's local fast path. - Rustrict, - /// The decisions API. - Jev, -} - -impl LabelSource { - /// The value stored in the `labels.source` column. - #[must_use] - pub fn as_str(self) -> &'static str { - match self { - Self::Rustrict => "rustrict", - Self::Jev => "jev", - } - } -} - -/// Labels for a record, and how they were decided. -#[derive(Debug, Clone)] -pub struct Inference { - /// The labels to store, including `!hide` or `!warn`. - pub labels: Vec, - /// Whether the labels came from rustrict or the decisions API. - pub source: LabelSource, -} - -/// Decide whether a record with the given content labels is hidden or shown -/// behind a warning. -fn action(content: &[LabelVal]) -> Option { - let hide = content - .iter() - .any(|label| matches!(label, LabelVal::Porn | LabelVal::Threat | LabelVal::Spam)); - - if hide { - Some(LabelVal::Hide) - } else if content.is_empty() { - None - } else { - Some(LabelVal::Warn) - } -} +mod verdict; pub struct ModeratorConfig { pub decisions_api_base_url: String, @@ -134,21 +55,21 @@ impl Moderator { /// Returns an error if the request fails, or the response is missing an /// answer to any question. #[instrument(skip(self, record))] - pub async fn infer_labels(&self, subject: &str, record: Value) -> Result { + pub async fn infer_labels(&self, subject: &str, record: Value) -> Result { let fast = fast_path::record_labels(&record); if !fast.is_empty() { debug!(?fast, "rustrict fast path applied"); - return Ok(Inference { - labels: fast, - source: LabelSource::Rustrict, - }); + return Ok(Verdict::new(fast, Source::Rustrict)); } let probabilities = self.judge(subject, record).await?; - Ok(Inference { - labels: labels(&probabilities), - source: LabelSource::Jev, - }) + let labels: Vec = probabilities + .iter() + .filter(|(_, p)| **p >= TRUE_THRESHOLD) + .map(|(l, _)| *l) + .collect(); + + Ok(Verdict::new(labels, Source::Jev)) } /// Ask the decisions API for the probability that each content label @@ -158,7 +79,7 @@ impl Moderator { /// /// Returns an error if the request fails, or the response is missing an /// answer to any question. - pub async fn judge(&self, subject: &str, record: Value) -> Result> { + async fn judge(&self, subject: &str, record: Value) -> Result> { let state = Value::Object(Map::from_iter([(subject.to_owned(), record)])); let questions = JUDGMENTS .iter() @@ -191,45 +112,10 @@ impl Moderator { } } -/// Add `!hide` or `!warn` to a set of content labels. -pub(crate) fn with_action(mut labels: Vec) -> Vec { - labels.extend(action(&labels)); - labels -} - -/// Decide the labels for a record from the probability of each content -/// label: the content labels judged true, plus `!hide` or `!warn`. -#[must_use] -pub fn labels(probabilities: &HashMap) -> Vec { - let labels: Vec = probabilities - .iter() - .filter(|(_, probability)| **probability > TRUE_THRESHOLD) - .map(|(label, _)| *label) - .collect(); - - with_action(labels) -} - #[cfg(test)] mod tests { use super::*; - #[test] - fn action_table() { - assert_eq!(action(&[]), None); - assert_eq!(action(&[LabelVal::Rude]), Some(LabelVal::Warn)); - assert_eq!( - action(&[LabelVal::Nudity, LabelVal::Sexual]), - Some(LabelVal::Warn) - ); - assert_eq!( - action(&[LabelVal::Rude, LabelVal::Threat]), - Some(LabelVal::Hide) - ); - assert_eq!(action(&[LabelVal::Spam]), Some(LabelVal::Hide)); - assert_eq!(action(&[LabelVal::Porn]), Some(LabelVal::Hide)); - } - #[test] fn questions_name_the_subject() { for judgment in &JUDGMENTS { diff --git a/crates/moderation/src/questions.rs b/crates/moderation/src/questions.rs index 50753ce..fd0463d 100644 --- a/crates/moderation/src/questions.rs +++ b/crates/moderation/src/questions.rs @@ -1,11 +1,11 @@ -use crate::LabelVal; +use crate::verdict::ContentLabel; /// Probability above which a judgment counts as true. pub(crate) const TRUE_THRESHOLD: f32 = 0.7; /// A yes/no judgment about a record that, when true, applies `label`. pub(crate) struct Judgment { - pub label: LabelVal, + pub label: ContentLabel, instructions: &'static str, when_true: &'static str, when_false: &'static str, @@ -29,43 +29,43 @@ impl Judgment { pub(crate) const JUDGMENTS: [Judgment; 7] = [ Judgment { - label: LabelVal::Porn, + label: ContentLabel::Porn, instructions: "Does `{subject}` contain pornographic content?", when_true: "The text describes sexual acts explicitly, or advertises or links to pornography.", when_false: "The text contains no explicit sexual descriptions and no links to pornography.", }, Judgment { - label: LabelVal::Sexual, + label: ContentLabel::Sexual, instructions: "Does `{subject}` contain sexually suggestive language?", when_true: "The text makes sexual jokes, innuendo, or suggestive remarks without describing sexual acts explicitly.", when_false: "The text makes no sexual jokes, innuendo, or suggestive remarks.", }, Judgment { - label: LabelVal::Nudity, + label: ContentLabel::Nudity, instructions: "Does `{subject}` refer to people being nude?", when_true: "The text mentions people being naked, such as naked hiking or a nude beach.", when_false: "The text does not mention anyone being unclothed.", }, Judgment { - label: LabelVal::GraphicMedia, + label: ContentLabel::GraphicMedia, instructions: "Does `{subject}` describe gore or violence in graphic detail?", when_true: "The text describes injuries, blood, or violence in vivid detail.", when_false: "The text mentions no injuries or violence, or mentions them only in passing, such as a warning about a steep drop.", }, Judgment { - label: LabelVal::Spam, + label: ContentLabel::Spam, instructions: "Is `{subject}` written to advertise a product, service, or website?", when_true: "The text promotes something for sale or directs readers to a commercial site.", when_false: "The text describes an activity, place, or person without promoting anything. Mentioning a café or shop along a route is not advertising.", }, Judgment { - label: LabelVal::Rude, + label: ContentLabel::Rude, instructions: "Does `{subject}` use insulting or offensive language?", when_true: "The text contains slurs, insults, or profanity aimed at people.", when_false: "The text is polite or neutral. Blunt criticism of a route, such as calling it boring or badly marked, is not rude.", }, Judgment { - label: LabelVal::Threat, + label: ContentLabel::Threat, instructions: "Does `{subject}` threaten or intimidate a person?", when_true: "The text threatens violence or harm against a person or group, or tries to intimidate them.", when_false: "The text contains no threats. Warnings about hazards such as traffic or wildlife are not threats.", diff --git a/crates/moderation/src/verdict.rs b/crates/moderation/src/verdict.rs new file mode 100644 index 0000000..252e4b9 --- /dev/null +++ b/crates/moderation/src/verdict.rs @@ -0,0 +1,137 @@ +use std::fmt::Display; + +#[derive(Ord, Eq, PartialEq, PartialOrd, Debug, Copy, Clone, Hash)] +pub enum ContentLabel { + Porn, + Sexual, + Nudity, + GraphicMedia, + Spam, + Rude, + Threat, +} + +impl ContentLabel { + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + ContentLabel::Porn => "porn", + ContentLabel::Sexual => "sexual", + ContentLabel::Nudity => "nudity", + ContentLabel::GraphicMedia => "graphic-media", + ContentLabel::Spam => "spam", + ContentLabel::Rude => "rude", + ContentLabel::Threat => "threat", + } + } +} + +impl Display for ContentLabel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Debug, PartialEq, Copy, Clone)] +pub enum Action { + Hide, + Warn, +} + +impl Action { + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::Hide => "!hide", + Self::Warn => "!warn", + } + } + + #[must_use] + pub fn for_labels(labels: &[ContentLabel]) -> Option { + if labels.is_empty() { + return None; + } + + for label in labels { + if let ContentLabel::Porn | ContentLabel::Threat | ContentLabel::Spam = label { + return Some(Self::Hide); + } + } + + Some(Self::Warn) + } +} + +pub enum Source { + Rustrict, + Jev, +} + +impl Source { + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Source::Rustrict => "rustrict", + Source::Jev => "jev", + } + } +} + +pub struct Verdict { + pub content: Vec, + pub action: Option, + pub source: Source, +} + +impl Verdict { + #[must_use] + pub fn new(mut content: Vec, source: Source) -> Self { + content.sort(); + let action = Action::for_labels(&content); + + Self { + content, + action, + source, + } + } + + #[must_use] + pub fn vals(&self) -> Vec<&'static str> { + self.content + .iter() + .map(|l| l.as_str()) + .chain(self.action.as_ref().map(|a| a.as_str())) + .collect() + } +} + +#[cfg(test)] +mod tests { + use crate::verdict::{Action, ContentLabel}; + + #[test] + fn empty_set_of_labels_should_result_in_no_action() { + let action = Action::for_labels(&[]); + assert_eq!(action, None); + } + + #[test] + fn non_severe_label_should_result_in_a_warning() { + let action = Action::for_labels(&[ContentLabel::Rude]); + assert_eq!(action, Some(Action::Warn)); + + let action = Action::for_labels(&[ContentLabel::Rude, ContentLabel::Sexual]); + assert_eq!(action, Some(Action::Warn)); + } + + #[test] + fn severe_label_should_result_in_hiding() { + let action = Action::for_labels(&[ContentLabel::Porn]); + assert_eq!(action, Some(Action::Hide)); + + let action = Action::for_labels(&[ContentLabel::Porn, ContentLabel::Threat]); + assert_eq!(action, Some(Action::Hide)); + } +} From c9e5c62701ac883330db91343e15746262266bdd Mon Sep 17 00:00:00 2001 From: Kshitij Chauhan Date: Fri, 6 Nov 2026 20:54:54 +0000 Subject: [PATCH 2/5] Introduce moderation subjects --- crates/ingest-consumer/src/main.rs | 20 ++--- crates/moderation/src/lib.rs | 19 ++-- crates/moderation/src/subject.rs | 137 +++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 18 deletions(-) create mode 100644 crates/moderation/src/subject.rs diff --git a/crates/ingest-consumer/src/main.rs b/crates/ingest-consumer/src/main.rs index aac139c..cb312e7 100644 --- a/crates/ingest-consumer/src/main.rs +++ b/crates/ingest-consumer/src/main.rs @@ -434,14 +434,11 @@ async fn consume_labels( return Ok(()); }; moderator - .infer_labels( - "route", - json!({ - "name": route.name, - "description": route.description, - "tags": route.tags, - }), - ) + .moderate(&moderation::Subject::Route { + name: route.name, + description: route.description, + tags: route.tags, + }) .await? } Some(ReviewRecord::NSID) => { @@ -460,7 +457,7 @@ async fn consume_labels( return Ok(()); }; moderator - .infer_labels("review", json!({ "text": body })) + .moderate(&moderation::Subject::Review { text: body }) .await? } Some(ProfileRecord::NSID) => { @@ -472,7 +469,10 @@ async fn consume_labels( return Ok(()); }; moderator - .infer_labels("profile", json!({ "name": user.name, "bio": user.bio })) + .moderate(&moderation::Subject::Profile { + name: user.name, + bio: user.bio, + }) .await? } _ => { diff --git a/crates/moderation/src/lib.rs b/crates/moderation/src/lib.rs index e8a2a32..0c1fa34 100644 --- a/crates/moderation/src/lib.rs +++ b/crates/moderation/src/lib.rs @@ -1,5 +1,4 @@ use anyhow::{Result, anyhow}; -use serde_json::{Map, Value}; use std::{collections::HashMap, time::Duration}; use tracing::{debug, instrument}; @@ -10,9 +9,12 @@ use crate::{ mod fast_path; mod questions; +mod subject; mod typesafe; mod verdict; +pub use subject::Subject; + pub struct ModeratorConfig { pub decisions_api_base_url: String, pub decisions_api_key: String, @@ -54,15 +56,16 @@ impl Moderator { /// /// Returns an error if the request fails, or the response is missing an /// answer to any question. - #[instrument(skip(self, record))] - pub async fn infer_labels(&self, subject: &str, record: Value) -> Result { - let fast = fast_path::record_labels(&record); + #[instrument(skip(self, subject))] + pub async fn moderate(&self, subject: &Subject) -> Result { + let state = subject.to_state(); + let fast = fast_path::record_labels(&state); if !fast.is_empty() { debug!(?fast, "rustrict fast path applied"); return Ok(Verdict::new(fast, Source::Rustrict)); } - let probabilities = self.judge(subject, record).await?; + let probabilities = self.judge(subject).await?; let labels: Vec = probabilities .iter() .filter(|(_, p)| **p >= TRUE_THRESHOLD) @@ -79,13 +82,13 @@ impl Moderator { /// /// Returns an error if the request fails, or the response is missing an /// answer to any question. - async fn judge(&self, subject: &str, record: Value) -> Result> { - let state = Value::Object(Map::from_iter([(subject.to_owned(), record)])); + async fn judge(&self, subject: &Subject) -> Result> { + let state = subject.to_state(); let questions = JUDGMENTS .iter() .map(|judgment| { let question = typesafe::Question::Noul { - instructions: judgment.instructions(subject), + instructions: judgment.instructions(subject.key()), criteria: typesafe::NoulCriteria { when_true: judgment.when_true(), when_false: judgment.when_false(), diff --git a/crates/moderation/src/subject.rs b/crates/moderation/src/subject.rs new file mode 100644 index 0000000..113a3ad --- /dev/null +++ b/crates/moderation/src/subject.rs @@ -0,0 +1,137 @@ +#[derive(PartialEq)] +pub enum Subject { + Route { + name: String, + description: Option, + tags: Vec, + }, + Review { + text: String, + }, + Profile { + name: String, + bio: Option, + }, +} + +impl Subject { + #[must_use] + pub(crate) fn key(&self) -> &'static str { + match self { + Subject::Route { .. } => "route", + Subject::Review { .. } => "review", + Subject::Profile { .. } => "profile", + } + } + + fn text_content(&self) -> Vec<&str> { + match self { + Subject::Route { + name, + description, + tags, + } => { + let mut texts: Vec<&str> = vec![name]; + if let Some(description) = description { + texts.push(description); + } + for tag in tags { + texts.push(tag); + } + texts + } + Subject::Review { text } => vec![text], + Subject::Profile { name, bio } => { + let mut texts: Vec<&str> = vec![name]; + if let Some(bio) = bio { + texts.push(bio); + } + + texts + } + } + } + + #[must_use] + pub(crate) fn to_state(&self) -> serde_json::Value { + match self { + Subject::Route { + name, + description, + tags, + } => serde_json::json!({ + "route": { + "name": name, + "description": description, + "tags": tags + } + }), + Subject::Review { text } => serde_json::json!({ + "review": { + "text": text + } + }), + Subject::Profile { name, bio } => serde_json::json!({ + "profile": { + "name": name, + "bio": bio, + } + }), + } + } +} + +#[cfg(test)] +mod tests { + use super::Subject; + + #[test] + fn review_state_should_accurately_describe_it() { + let review = Subject::Review { + text: String::from("test"), + }; + + assert_eq!( + review.to_state(), + serde_json::json!({"review": { "text": "test" }}) + ); + } + + #[test] + fn route_state_should_accurately_describe_it() { + let route = Subject::Route { + name: String::from("test-route"), + description: Some(String::from("test-description")), + tags: vec![String::from("test-tag")], + }; + + assert_eq!( + route.to_state(), + serde_json::json!({ + "route": { + "name": "test-route", + "description": "test-description", + "tags": ["test-tag"] + } + }) + ); + } + + #[test] + fn profile_state_should_accurately_describe_it() { + let profile = Subject::Profile { + name: String::from("test-name"), + bio: Some(String::from("test-bio")), + }; + + assert_eq!( + profile.to_state(), + serde_json::json!({ + "profile": { + "name": "test-name", + "bio": "test-bio", + } + }) + ); + } +} From 145f1feed9af513e92053ae28d6ea03dac7ea417 Mon Sep 17 00:00:00 2001 From: Kshitij Chauhan Date: Fri, 6 Nov 2026 21:28:54 +0000 Subject: [PATCH 3/5] De-sloppify the rustrict code --- crates/ingest-consumer/src/main.rs | 1 - crates/moderation/src/fast_path.rs | 187 ----------------------------- crates/moderation/src/lib.rs | 10 +- crates/moderation/src/rustrict.rs | 119 ++++++++++++++++++ crates/moderation/src/subject.rs | 2 +- 5 files changed, 124 insertions(+), 195 deletions(-) delete mode 100644 crates/moderation/src/fast_path.rs create mode 100644 crates/moderation/src/rustrict.rs diff --git a/crates/ingest-consumer/src/main.rs b/crates/ingest-consumer/src/main.rs index cb312e7..f6d574c 100644 --- a/crates/ingest-consumer/src/main.rs +++ b/crates/ingest-consumer/src/main.rs @@ -25,7 +25,6 @@ use lifecycle::{ shutdown::Shutdown, }; use moderation::{Moderator, ModeratorConfig}; -use serde_json::json; use tokio::time::timeout; use tokio_util::sync::CancellationToken; use tracing::{error, info, info_span, warn}; diff --git a/crates/moderation/src/fast_path.rs b/crates/moderation/src/fast_path.rs deleted file mode 100644 index 266ff1e..0000000 --- a/crates/moderation/src/fast_path.rs +++ /dev/null @@ -1,187 +0,0 @@ -use rustrict::{Censor, Type}; -use serde_json::Value; - -use crate::verdict::ContentLabel; - -/// Only terms rustrict rates as severe may bypass the decisions API. -const MIN_SEVERITY: Type = Type::SEVERE; - -/// A function that returns labels for text, or `None` if the result is -/// ambiguous and must be decided by the model. -type LabelCheck = fn(&str) -> Option>; - -/// Map a rustrict analysis to labels without involving any real text. -/// -/// Severe `mean` matches are ambiguous and return `None`: rustrict uses `mean` -/// for both ordinary insults and violent language, which need different labels. -fn labels_for_analysis(analysis: Type) -> Option> { - if analysis.is(Type::MEAN & MIN_SEVERITY) { - return None; - } - - let mut labels = Vec::new(); - if analysis.is((Type::PROFANE | Type::OFFENSIVE) & MIN_SEVERITY) { - labels.push(ContentLabel::Rude); - } - if analysis.is(Type::SEXUAL & MIN_SEVERITY) { - labels.push(ContentLabel::Porn); - } - - Some(labels) -} - -/// Labels rustrict can decide for one piece of text. -/// -/// Returns `None` when the text contains an ambiguous severe `mean` match. -#[must_use] -fn text_labels(text: &str) -> Option> { - if text.trim().is_empty() { - return Some(Vec::new()); - } - - let mut censor = Censor::from_str(text); - censor - .with_ignore_false_positives(false) - .with_ignore_self_censoring(false); - - labels_for_analysis(censor.analyze()) -} - -/// Labels for every string value in a record, including `!hide`/`!warn`. -/// -/// Object keys are not inspected. Any severe match in any field is enough for -/// the whole record to skip the decisions API, unless any field has an -/// ambiguous severe `mean` match. -#[must_use] -pub(crate) fn record_labels(record: &Value) -> Vec { - let mut labels = Vec::new(); - if collect(record, &mut labels, text_labels) { - return Vec::new(); - } - - labels -} - -/// Collect labels, returning `true` if any value is ambiguous. -fn collect(value: &Value, labels: &mut Vec, check: LabelCheck) -> bool { - match value { - Value::String(text) => match check(text) { - Some(found) => { - for label in found { - if !labels.contains(&label) { - labels.push(label); - } - } - false - } - None => true, - }, - Value::Array(values) => values.iter().any(|value| collect(value, labels, check)), - Value::Object(map) => map.values().any(|value| collect(value, labels, check)), - Value::Null | Value::Bool(_) | Value::Number(_) => false, - } -} - -#[cfg(test)] -mod tests { - use serde_json::json; - - use super::*; - - #[test] - fn severe_profanity_is_rude() { - assert_eq!( - labels_for_analysis(Type::PROFANE & Type::SEVERE), - Some(vec![ContentLabel::Rude]) - ); - } - - #[test] - fn severe_offensive_is_rude() { - assert_eq!( - labels_for_analysis(Type::OFFENSIVE & Type::SEVERE), - Some(vec![ContentLabel::Rude]) - ); - } - - #[test] - fn severe_sexual_is_porn() { - assert_eq!( - labels_for_analysis(Type::SEXUAL & Type::SEVERE), - Some(vec![ContentLabel::Porn]) - ); - } - - #[test] - fn moderate_categories_are_not_fast_pathed() { - assert_eq!( - labels_for_analysis(Type::PROFANE & Type::MODERATE), - Some(Vec::new()) - ); - assert_eq!( - labels_for_analysis(Type::SEXUAL & Type::MODERATE), - Some(Vec::new()) - ); - } - - #[test] - fn severe_mean_is_ambiguous() { - assert_eq!(labels_for_analysis(Type::MEAN & Type::SEVERE), None); - assert_eq!( - labels_for_analysis((Type::PROFANE | Type::MEAN) & Type::SEVERE), - None - ); - } - - #[test] - fn benign_text_is_not_flagged() { - assert_eq!(text_labels("A quiet walk by the river"), Some(Vec::new())); - assert_eq!(text_labels(""), Some(Vec::new())); - } - - /// A stand-in for `text_labels` so recursion can be tested without - /// embedding real moderation terms in the source. - fn fake_check(text: &str) -> Option> { - match text { - "flagged" => Some(vec![ContentLabel::Rude]), - "unclear" => None, - _ => Some(Vec::new()), - } - } - - #[test] - fn collect_checks_nested_values() { - let mut labels = Vec::new(); - - assert!(!collect( - &json!({ "tags": ["hiking", "flagged"] }), - &mut labels, - fake_check - )); - assert_eq!(labels, vec![ContentLabel::Rude]); - } - - #[test] - fn ambiguous_value_makes_the_whole_record_fall_through() { - let mut labels = Vec::new(); - - assert!(collect( - &json!({ "name": "flagged", "bio": "unclear" }), - &mut labels, - fake_check - )); - } - - #[test] - fn benign_record_has_no_labels() { - assert!( - record_labels(&json!({ - "name": "Ben Nevis", - "description": "A steep climb with a warning about the drop", - "tags": ["munro", "scotland"], - })) - .is_empty() - ); - assert!(record_labels(&json!({ "text": "A quiet walk by the river" })).is_empty()); - } -} diff --git a/crates/moderation/src/lib.rs b/crates/moderation/src/lib.rs index 0c1fa34..35a4ea4 100644 --- a/crates/moderation/src/lib.rs +++ b/crates/moderation/src/lib.rs @@ -7,8 +7,8 @@ use crate::{ verdict::{ContentLabel, Source, Verdict}, }; -mod fast_path; mod questions; +mod rustrict; mod subject; mod typesafe; mod verdict; @@ -58,11 +58,9 @@ impl Moderator { /// answer to any question. #[instrument(skip(self, subject))] pub async fn moderate(&self, subject: &Subject) -> Result { - let state = subject.to_state(); - let fast = fast_path::record_labels(&state); - if !fast.is_empty() { - debug!(?fast, "rustrict fast path applied"); - return Ok(Verdict::new(fast, Source::Rustrict)); + let rustrict_labels = rustrict::screen(&subject.text_content()); + if let Some(labels) = rustrict_labels { + return Ok(Verdict::new(labels, Source::Rustrict)); } let probabilities = self.judge(subject).await?; diff --git a/crates/moderation/src/rustrict.rs b/crates/moderation/src/rustrict.rs new file mode 100644 index 0000000..bdba2c9 --- /dev/null +++ b/crates/moderation/src/rustrict.rs @@ -0,0 +1,119 @@ +use rustrict::{Censor, Type}; + +use crate::verdict::ContentLabel; + +/// Only terms rustrict rates as severe may bypass the decisions API. +const MIN_SEVERITY: Type = Type::SEVERE; + +fn analyze(text: &str) -> Type { + let mut censor = Censor::from_str(text); + censor.analyze() +} + +fn to_labels(analysis: Type) -> Option> { + if analysis.is(Type::NONE) { + return None; + } + + if analysis.is(Type::MEAN & MIN_SEVERITY) { + return None; + } + + let mut labels = Vec::new(); + if analysis.is((Type::PROFANE | Type::OFFENSIVE) & MIN_SEVERITY) { + labels.push(ContentLabel::Rude); + } + + if analysis.is(Type::SEXUAL & MIN_SEVERITY) { + labels.push(ContentLabel::Porn); + } + + if labels.is_empty() { + None + } else { + Some(labels) + } +} + +/// Screens a given collection of text strings for profanity +/// and offensive content using rustrict. +/// +/// A return value of `None` from this function must be considered +/// ambiguous, as it only implies that rustrict couldn't find an +/// obvious problem. +#[must_use] +pub fn screen(texts: &[&str]) -> Option> { + let analysis = texts + .iter() + .fold(Type::NONE, |all, text| all | analyze(text)); + + to_labels(analysis) +} + +#[cfg(test)] +mod tests { + + use crate::{Subject, verdict::ContentLabel}; + + use super::{analyze, screen, to_labels}; + use rustrict::Type; + + #[test] + fn empty_text_is_not_censored() { + assert_eq!(analyze(""), Type::NONE); + } + + #[test] + fn severe_profanity_is_rude() { + assert_eq!( + to_labels(Type::PROFANE & Type::SEVERE), + Some(vec![ContentLabel::Rude]) + ); + } + + #[test] + fn severe_offensive_is_rude() { + assert_eq!( + to_labels(Type::OFFENSIVE & Type::SEVERE), + Some(vec![ContentLabel::Rude]) + ); + } + + #[test] + fn severe_sexual_is_porn() { + assert_eq!( + to_labels(Type::SEXUAL & Type::SEVERE), + Some(vec![ContentLabel::Porn]) + ); + } + + #[test] + fn moderate_categories_produce_no_labels() { + assert_eq!(to_labels(Type::PROFANE & Type::MODERATE), None); + assert_eq!(to_labels(Type::SEXUAL & Type::MODERATE), None); + } + + #[test] + fn severe_mean_is_ambiguous() { + assert_eq!(to_labels(Type::MEAN & Type::SEVERE), None); + assert_eq!(to_labels((Type::PROFANE | Type::MEAN) & Type::SEVERE), None); + } + + #[test] + fn benign_text_is_not_flagged() { + assert_eq!(screen(&["A quiet walk by the river"]), None); + assert_eq!(screen(&[]), None); + assert_eq!(screen(&[""]), None); + } + + #[test] + fn benign_record_has_no_labels() { + let subject = Subject::Route { + name: String::from("Ben Nevis"), + description: Some(String::from("A steep climb with a warning about the drop")), + tags: vec![String::from("scotland")], + }; + + assert_eq!(screen(&subject.text_content()), None); + } +} diff --git a/crates/moderation/src/subject.rs b/crates/moderation/src/subject.rs index 113a3ad..c26e4cb 100644 --- a/crates/moderation/src/subject.rs +++ b/crates/moderation/src/subject.rs @@ -24,7 +24,7 @@ impl Subject { } } - fn text_content(&self) -> Vec<&str> { + pub(crate) fn text_content(&self) -> Vec<&str> { match self { Subject::Route { name, From 5f553f2429579bd19bfa91e4bce238ec666b0848 Mon Sep 17 00:00:00 2001 From: Kshitij Chauhan Date: Fri, 6 Nov 2026 22:02:06 +0000 Subject: [PATCH 4/5] Generate typesafe ai types from openapi spec --- crates/moderation/src/lib.rs | 19 +- crates/moderation/src/questions.rs | 2 +- crates/moderation/src/subject.rs | 39 +- crates/moderation/src/typesafe/generated.rs | 540 ++++++++++++++++++ .../src/{typesafe.rs => typesafe/mod.rs} | 67 +-- crates/moderation/typesafe/openapi.json | 1 + mise.toml | 12 + 7 files changed, 610 insertions(+), 70 deletions(-) create mode 100644 crates/moderation/src/typesafe/generated.rs rename crates/moderation/src/{typesafe.rs => typesafe/mod.rs} (65%) create mode 100644 crates/moderation/typesafe/openapi.json diff --git a/crates/moderation/src/lib.rs b/crates/moderation/src/lib.rs index 35a4ea4..ed25515 100644 --- a/crates/moderation/src/lib.rs +++ b/crates/moderation/src/lib.rs @@ -80,18 +80,16 @@ impl Moderator { /// /// Returns an error if the request fails, or the response is missing an /// answer to any question. - async fn judge(&self, subject: &Subject) -> Result> { - let state = subject.to_state(); + async fn judge(&self, subject: &Subject) -> Result> { + let state = typesafe::State::Object(subject.to_state()); let questions = JUDGMENTS .iter() .map(|judgment| { - let question = typesafe::Question::Noul { - instructions: judgment.instructions(subject.key()), - criteria: typesafe::NoulCriteria { - when_true: judgment.when_true(), - when_false: judgment.when_false(), - }, - }; + let question = typesafe::noul( + judgment.instructions(subject.key()), + String::from(judgment.when_true()), + String::from(judgment.when_false()), + ); (judgment.label.as_str().to_owned(), question) }) .collect(); @@ -100,7 +98,8 @@ impl Moderator { let probabilities = JUDGMENTS .iter() .map(|judgment| match answers.get(judgment.label.as_str()) { - Some(typesafe::Answer::Noul { noul }) => Ok((judgment.label, *noul)), + Some(typesafe::Answer::NoulAnswer(noul)) => Ok((judgment.label, noul.noul)), + Some(_) => Err(anyhow!("expected noul answer")), None => Err(anyhow!( "no answer for {} in decisions api response", judgment.label diff --git a/crates/moderation/src/questions.rs b/crates/moderation/src/questions.rs index fd0463d..690e525 100644 --- a/crates/moderation/src/questions.rs +++ b/crates/moderation/src/questions.rs @@ -1,7 +1,7 @@ use crate::verdict::ContentLabel; /// Probability above which a judgment counts as true. -pub(crate) const TRUE_THRESHOLD: f32 = 0.7; +pub(crate) const TRUE_THRESHOLD: f64 = 0.7; /// A yes/no judgment about a record that, when true, applies `label`. pub(crate) struct Judgment { diff --git a/crates/moderation/src/subject.rs b/crates/moderation/src/subject.rs index c26e4cb..fb86dbd 100644 --- a/crates/moderation/src/subject.rs +++ b/crates/moderation/src/subject.rs @@ -53,30 +53,33 @@ impl Subject { } #[must_use] - pub(crate) fn to_state(&self) -> serde_json::Value { + pub(crate) fn to_state(&self) -> serde_json::Map { match self { Subject::Route { name, description, tags, - } => serde_json::json!({ - "route": { + } => serde_json::map::Map::from_iter([( + String::from("route"), + serde_json::json!({ "name": name, "description": description, "tags": tags - } - }), - Subject::Review { text } => serde_json::json!({ - "review": { + }), + )]), + Subject::Review { text } => serde_json::map::Map::from_iter([( + String::from("review"), + serde_json::json!({ "text": text - } - }), - Subject::Profile { name, bio } => serde_json::json!({ - "profile": { - "name": name, - "bio": bio, - } - }), + }), + )]), + Subject::Profile { name, bio } => serde_json::map::Map::from_iter([( + String::from("profile"), + serde_json::json!({ + "name": name, + "bio": bio, + }), + )]), } } } @@ -92,7 +95,7 @@ mod tests { }; assert_eq!( - review.to_state(), + serde_json::to_value(review.to_state()).expect("failed to serialise value"), serde_json::json!({"review": { "text": "test" }}) ); } @@ -106,7 +109,7 @@ mod tests { }; assert_eq!( - route.to_state(), + serde_json::to_value(route.to_state()).expect("failed to serialise value"), serde_json::json!({ "route": { "name": "test-route", @@ -125,7 +128,7 @@ mod tests { }; assert_eq!( - profile.to_state(), + serde_json::to_value(profile.to_state()).expect("failed to serialise value"), serde_json::json!({ "profile": { "name": "test-name", diff --git a/crates/moderation/src/typesafe/generated.rs b/crates/moderation/src/typesafe/generated.rs new file mode 100644 index 0000000..71ec25e --- /dev/null +++ b/crates/moderation/src/typesafe/generated.rs @@ -0,0 +1,540 @@ +#![allow(clippy::redundant_closure_call)] +#![allow(clippy::needless_lifetimes)] +#![allow(clippy::match_single_binding)] +#![allow(clippy::clone_on_copy)] + +#[doc = "An answer whose type matches the corresponding question."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +#[serde(untagged)] +pub enum Answer { + NoulAnswer(NoulAnswer), + ChoiceAnswer(ChoiceAnswer), + ScoreAnswer(ScoreAnswer), + BoxAnswer(BoxAnswer), +} +impl ::std::convert::From for Answer { + fn from(value: NoulAnswer) -> Self { + Self::NoulAnswer(value) + } +} +impl ::std::convert::From for Answer { + fn from(value: ChoiceAnswer) -> Self { + Self::ChoiceAnswer(value) + } +} +impl ::std::convert::From for Answer { + fn from(value: ScoreAnswer) -> Self { + Self::ScoreAnswer(value) + } +} +impl ::std::convert::From for Answer { + fn from(value: BoxAnswer) -> Self { + Self::BoxAnswer(value) + } +} +#[doc = "An image that replaces its placeholder: its bytes in data_base64 with their format, or, in a multipart request, the name of the image part that holds them. Requests with images must send the header TypeSafe-Danger-Zone-Images: true."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug, Default)] +pub struct Asset { + #[doc = "Base64-encoded image data."] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub data_base64: ::std::option::Option<::std::string::String>, + #[doc = "The image format, required with data_base64. The actual format is detected from the bytes."] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub format: ::std::option::Option, + #[doc = "In a multipart request, the name of the image part this asset uses. With part, data_base64 is ignored."] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub part: ::std::option::Option<::std::string::String>, + #[doc = "Identifies an image asset."] + #[serde( + rename = "type", + skip_serializing_if = "::std::option::Option::is_none" + )] + pub type_: ::std::option::Option<::std::string::String>, +} +#[doc = "A located box with normalized edge coordinates and confidence."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct BoxAnswer { + #[doc = "The box's edges as fractions of image height and width, from 0 to 1. Each coordinate is the expected value of its edge's predicted position."] + #[serde(rename = "box")] + pub box_: BoxCoordinates, + #[doc = "Confidence for the given box, from 0 to 1, with 1 being more confident. Null when the model does not provide confidence."] + #[serde(deserialize_with = "::std::option::Option::deserialize")] + pub confidence: ::std::option::Option, + #[doc = "Identifies a box answer."] + #[serde(rename = "type")] + pub type_: ::std::string::String, +} +#[doc = "Normalized box edges. Vertical coordinates are measured from the top of the image; horizontal coordinates are measured from the left."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct BoxCoordinates { + #[doc = "Bottom edge of the box, as a fraction of image height from the top (0-1)."] + pub bottom: f64, + #[doc = "Left edge of the box, as a fraction of image width from the left (0-1)."] + pub left: f64, + #[doc = "Right edge of the box, as a fraction of image width from the left (0-1)."] + pub right: f64, + #[doc = "Top edge of the box, as a fraction of image height from the top (0-1)."] + pub top: f64, +} +#[doc = "A question that locates what its instructions describe in the request's image. Requests with box questions must send the header TypeSafe-Danger-Zone-Images: true."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct BoxQuestion { + #[doc = "What to locate in the image."] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub instructions: ::std::option::Option, + #[doc = "Identifies a box question."] + #[serde(rename = "type")] + pub type_: ::std::string::String, +} +#[doc = "The selected choice, confidence, and probabilities for a choice question."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct ChoiceAnswer { + #[doc = "The name of the choice with the highest probability among the question's criteria."] + pub choice: ::std::string::String, + #[doc = "Confidence in the selected choice, from 0 to 1. Higher values indicate greater certainty; use lower values to flag uncertain selections for review."] + pub confidence: f64, + #[doc = "Probability of each choice in criteria, keyed by choice name, from 0 to 1. Shows how likely the alternatives are; values sum to approximately 1."] + pub probabilities: ::std::collections::HashMap<::std::string::String, f64>, + #[doc = "Identifies a selection from the requested choices."] + #[serde(rename = "type")] + pub type_: ::std::string::String, +} +#[doc = "A question that selects one option from the choices you define."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct ChoiceQuestion { + #[doc = "Choice names and descriptions of when each applies. A choice without a description is interpreted by its name alone."] + pub criteria: ::std::collections::HashMap<::std::string::String, ChoiceQuestionCriteriaValue>, + #[doc = "What the model should decide when choosing an option."] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub instructions: ::std::option::Option, + #[doc = "Identifies a question that selects one of the choices in criteria."] + #[serde(rename = "type")] + pub type_: ::std::string::String, +} +#[doc = "`ChoiceQuestionCriteriaValue`"] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +#[serde(untagged)] +pub enum ChoiceQuestionCriteriaValue { + String(::std::string::String), + Object(::serde_json::Map<::std::string::String, ::serde_json::Value>), + Array(::std::vec::Vec<::serde_json::Value>), + Null, +} +impl ::std::convert::From<::serde_json::Map<::std::string::String, ::serde_json::Value>> + for ChoiceQuestionCriteriaValue +{ + fn from(value: ::serde_json::Map<::std::string::String, ::serde_json::Value>) -> Self { + Self::Object(value) + } +} +impl ::std::convert::From<::std::vec::Vec<::serde_json::Value>> for ChoiceQuestionCriteriaValue { + fn from(value: ::std::vec::Vec<::serde_json::Value>) -> Self { + Self::Array(value) + } +} +#[doc = "`CriteriaItem`"] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +#[serde(untagged)] +pub enum CriteriaItem { + String(::std::string::String), + Object(::serde_json::Map<::std::string::String, ::serde_json::Value>), + Array(::std::vec::Vec<::serde_json::Value>), +} +impl ::std::convert::From<::serde_json::Map<::std::string::String, ::serde_json::Value>> + for CriteriaItem +{ + fn from(value: ::serde_json::Map<::std::string::String, ::serde_json::Value>) -> Self { + Self::Object(value) + } +} +impl ::std::convert::From<::std::vec::Vec<::serde_json::Value>> for CriteriaItem { + fn from(value: ::std::vec::Vec<::serde_json::Value>) -> Self { + Self::Array(value) + } +} +#[doc = "Explanation of the invalid request. String details and message fields are bounded, with shortened text ending with …."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +#[serde(untagged)] +pub enum Detail { + String(::std::string::String), + Object(::serde_json::Map<::std::string::String, ::serde_json::Value>), +} +impl ::std::convert::From<::serde_json::Map<::std::string::String, ::serde_json::Value>> + for Detail +{ + fn from(value: ::serde_json::Map<::std::string::String, ::serde_json::Value>) -> Self { + Self::Object(value) + } +} +#[doc = "What counts as a no answer."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +#[serde(untagged)] +pub enum False { + String(::std::string::String), + Object(::serde_json::Map<::std::string::String, ::serde_json::Value>), + Array(::std::vec::Vec<::serde_json::Value>), + Null, +} +impl ::std::convert::From<::serde_json::Map<::std::string::String, ::serde_json::Value>> for False { + fn from(value: ::serde_json::Map<::std::string::String, ::serde_json::Value>) -> Self { + Self::Object(value) + } +} +impl ::std::convert::From<::std::vec::Vec<::serde_json::Value>> for False { + fn from(value: ::std::vec::Vec<::serde_json::Value>) -> Self { + Self::Array(value) + } +} +#[doc = "The image format, required with data_base64. The actual format is detected from the bytes."] +#[derive( + :: serde :: Deserialize, + :: serde :: Serialize, + Clone, + Copy, + Debug, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, +)] +pub enum Format { + #[serde(rename = "png")] + Png, + #[serde(rename = "jpg")] + Jpg, + #[serde(rename = "jpeg")] + Jpeg, +} +impl ::std::fmt::Display for Format { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + match *self { + Self::Png => f.write_str("png"), + Self::Jpg => f.write_str("jpg"), + Self::Jpeg => f.write_str("jpeg"), + } + } +} +impl ::std::str::FromStr for Format { + type Err = self::error::ConversionError; + fn from_str(value: &str) -> ::std::result::Result { + match value { + "png" => Ok(Self::Png), + "jpg" => Ok(Self::Jpg), + "jpeg" => Ok(Self::Jpeg), + _ => Err("invalid value".into()), + } + } +} +impl ::std::convert::TryFrom<&str> for Format { + type Error = self::error::ConversionError; + fn try_from(value: &str) -> ::std::result::Result { + value.parse() + } +} +impl ::std::convert::TryFrom<::std::string::String> for Format { + type Error = self::error::ConversionError; + fn try_from( + value: ::std::string::String, + ) -> ::std::result::Result { + value.parse() + } +} +#[doc = "Application validation failures returned with HTTP status 400."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct HttpBadRequestError { + #[doc = "Explanation of the invalid request. String details and message fields are bounded, with shortened text ending with …."] + pub detail: Detail, +} +#[doc = "Request validation failures returned with HTTP status 422."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug, Default)] +pub struct HttpValidationError { + #[doc = "Validation errors describing which request values are missing or invalid."] + #[serde(default, skip_serializing_if = "::std::vec::Vec::is_empty")] + pub detail: ::std::vec::Vec, +} +#[doc = "What to locate in the image."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +#[serde(untagged)] +pub enum Instructions { + String(::std::string::String), + Object(::serde_json::Map<::std::string::String, ::serde_json::Value>), + Array(::std::vec::Vec<::serde_json::Value>), + Null, +} +impl ::std::convert::From<::serde_json::Map<::std::string::String, ::serde_json::Value>> + for Instructions +{ + fn from(value: ::serde_json::Map<::std::string::String, ::serde_json::Value>) -> Self { + Self::Object(value) + } +} +impl ::std::convert::From<::std::vec::Vec<::serde_json::Value>> for Instructions { + fn from(value: ::std::vec::Vec<::serde_json::Value>) -> Self { + Self::Array(value) + } +} +#[doc = "`LocationItem`"] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +#[serde(untagged)] +pub enum LocationItem { + String(::std::string::String), + Integer(i64), +} +impl ::std::fmt::Display for LocationItem { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + match self { + Self::String(x) => x.fmt(f), + Self::Integer(x) => x.fmt(f), + } + } +} +impl ::std::convert::From for LocationItem { + fn from(value: i64) -> Self { + Self::Integer(value) + } +} +#[doc = "A model or model alias available to the authenticated account."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct ModelMetadata { + #[doc = "Human-readable description of the model and its capabilities."] + pub description: ::std::string::String, + #[doc = "Model name or alias accepted by the request's model field."] + pub name: ::std::string::String, + #[doc = "Model release date, formatted as YYYY-MM-DD."] + pub release_date: ::std::string::String, +} +#[doc = "Models and aliases available to the authenticated account."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct ModelMetadataList { + #[doc = "Available models and aliases. Use a model's name in POST /v1/systemone requests."] + pub models: ::std::vec::Vec, +} +#[doc = "The probability of a yes answer or a true statement."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct NoulAnswer { + #[doc = "Probability of a yes answer or a true statement, from 0 to 1. Values near 1 favor yes or true, values near 0 favor no or false, and values near 0.5 indicate uncertainty."] + pub noul: f64, + #[doc = "Identifies a yes/no answer."] + #[serde(rename = "type")] + pub type_: ::std::string::String, +} +#[doc = "Criteria defining what counts as a yes or no answer."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug, Default)] +pub struct NoulCriteria { + #[doc = "What counts as a no answer."] + #[serde( + rename = "false", + skip_serializing_if = "::std::option::Option::is_none" + )] + pub false_: ::std::option::Option, + #[doc = "What counts as a yes answer."] + #[serde( + rename = "true", + skip_serializing_if = "::std::option::Option::is_none" + )] + pub true_: ::std::option::Option, +} +#[doc = "A yes/no question or statement, answered with the probability of yes or true."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct NoulQuestion { + #[doc = "Criteria clarifying what counts as a yes or no answer."] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub criteria: ::std::option::Option, + #[doc = "The yes/no question or statement to evaluate."] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub instructions: ::std::option::Option, + #[doc = "Identifies a yes/no question or statement."] + #[serde(rename = "type")] + pub type_: ::std::string::String, +} +#[doc = "A question about the supplied content."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +#[serde(untagged)] +pub enum Question { + NoulQuestion(NoulQuestion), + ChoiceQuestion(ChoiceQuestion), + ScoreQuestion(ScoreQuestion), + BoxQuestion(BoxQuestion), +} +impl ::std::convert::From for Question { + fn from(value: NoulQuestion) -> Self { + Self::NoulQuestion(value) + } +} +impl ::std::convert::From for Question { + fn from(value: ChoiceQuestion) -> Self { + Self::ChoiceQuestion(value) + } +} +impl ::std::convert::From for Question { + fn from(value: ScoreQuestion) -> Self { + Self::ScoreQuestion(value) + } +} +impl ::std::convert::From for Question { + fn from(value: BoxQuestion) -> Self { + Self::BoxQuestion(value) + } +} +#[doc = "An expected score with its rubric, confidence, and score-level probabilities."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct ScoreAnswer { + #[doc = "Confidence in the score, from 0 to 1. Higher values indicate greater certainty; use lower values to flag uncertain ratings for review."] + pub confidence: f64, + #[doc = "The requested criteria mapped to their score levels, so you can interpret the score."] + pub legend: ::std::collections::HashMap<::std::string::String, ScoreAnswerLegendValue>, + #[doc = "Probability of each score level, from 0 to 1, using the same keys as legend. Shows how likely the alternatives are; values sum to approximately 1."] + pub probabilities: ::std::collections::HashMap<::std::string::String, f64>, + #[doc = "Expected score: the probability-weighted average of the rubric levels. May fall between integer levels."] + pub score: f64, + #[doc = "Identifies a rating against the requested score levels."] + #[serde(rename = "type")] + pub type_: ::std::string::String, +} +#[doc = "`ScoreAnswerLegendValue`"] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +#[serde(untagged)] +pub enum ScoreAnswerLegendValue { + String(::std::string::String), + Object(::serde_json::Map<::std::string::String, ::serde_json::Value>), + Array(::std::vec::Vec<::serde_json::Value>), +} +impl ::std::convert::From<::serde_json::Map<::std::string::String, ::serde_json::Value>> + for ScoreAnswerLegendValue +{ + fn from(value: ::serde_json::Map<::std::string::String, ::serde_json::Value>) -> Self { + Self::Object(value) + } +} +impl ::std::convert::From<::std::vec::Vec<::serde_json::Value>> for ScoreAnswerLegendValue { + fn from(value: ::std::vec::Vec<::serde_json::Value>) -> Self { + Self::Array(value) + } +} +#[doc = "A question that assigns a score using an ordered rubric."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct ScoreQuestion { + #[doc = "Ordered descriptions of the score levels. Each description's position determines its score, starting at zero."] + pub criteria: ::std::vec::Vec, + #[doc = "What the model should rate."] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub instructions: ::std::option::Option, + #[doc = "Identifies a question that rates the content using the levels in criteria."] + #[serde(rename = "type")] + pub type_: ::std::string::String, +} +#[doc = "The content all questions in this request refer to."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +#[serde(untagged)] +pub enum State { + String(::std::string::String), + Object(::serde_json::Map<::std::string::String, ::serde_json::Value>), + Array(::std::vec::Vec<::serde_json::Value>), +} +impl ::std::convert::From<::serde_json::Map<::std::string::String, ::serde_json::Value>> for State { + fn from(value: ::serde_json::Map<::std::string::String, ::serde_json::Value>) -> Self { + Self::Object(value) + } +} +impl ::std::convert::From<::std::vec::Vec<::serde_json::Value>> for State { + fn from(value: ::std::vec::Vec<::serde_json::Value>) -> Self { + Self::Array(value) + } +} +#[doc = "Content and named questions to evaluate together using a TypeSafe model."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct SystemOneRequest { + #[doc = "Media assets keyed by an opaque placeholder (recommend a per-image UUID). Each placeholder, wherever it appears as or inside a string in state, question instructions, or criteria, is replaced with the asset. Image assets require a vision-language model."] + #[serde( + default, + skip_serializing_if = ":: std :: collections :: HashMap::is_empty" + )] + pub assets: ::std::collections::HashMap<::std::string::String, Asset>, + #[doc = "Name or alias of the model to use. Available names are returned by GET /v1/models."] + pub model: ::std::string::String, + #[doc = "Questions to ask about the content, each with a name you choose. The response uses those names to identify the answers."] + pub questions: ::std::collections::HashMap<::std::string::String, Question>, + #[doc = "The content all questions in this request refer to."] + pub state: State, +} +#[doc = "Answers grouped by question name, with the model used and token usage."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct SystemOneResponse { + #[doc = "Answers keyed by the question names supplied in the request. Each answer's type matches its question's type."] + pub answers: ::std::collections::HashMap<::std::string::String, Answer>, + #[doc = "Name of the model that answered the questions. May differ from the alias supplied in the request."] + pub model: ::std::string::String, + #[doc = "Input and output token counts for this evaluation."] + pub usage: Usage, +} +#[doc = "What counts as a yes answer."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +#[serde(untagged)] +pub enum True { + String(::std::string::String), + Object(::serde_json::Map<::std::string::String, ::serde_json::Value>), + Array(::std::vec::Vec<::serde_json::Value>), + Null, +} +impl ::std::convert::From<::serde_json::Map<::std::string::String, ::serde_json::Value>> for True { + fn from(value: ::serde_json::Map<::std::string::String, ::serde_json::Value>) -> Self { + Self::Object(value) + } +} +impl ::std::convert::From<::std::vec::Vec<::serde_json::Value>> for True { + fn from(value: ::std::vec::Vec<::serde_json::Value>) -> Self { + Self::Array(value) + } +} +#[doc = "Token usage for the request."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct Usage { + #[doc = "Number of billable input tokens used to evaluate the request."] + pub input_tokens: i64, + #[doc = "Number of output tokens used to answer the questions. Output tokens are currently free of charge."] + pub output_tokens: i64, +} +#[doc = "A request validation error at a specific field or array element."] +#[derive(:: serde :: Deserialize, :: serde :: Serialize, Clone, Debug)] +pub struct ValidationError { + #[doc = "Additional context used to explain the validation failure. Entries may be omitted or shortened."] + #[serde(default, skip_serializing_if = "::serde_json::Map::is_empty")] + pub ctx: ::serde_json::Map<::std::string::String, ::serde_json::Value>, + #[doc = "Text preview of the rejected input. May be shortened with …."] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub input: ::std::option::Option<::std::string::String>, + #[doc = "Path to the invalid value: the request location followed by field names and array indices. May be empty."] + pub loc: ::std::vec::Vec, + #[doc = "Human-readable explanation of the validation failure. May be shortened with …."] + pub msg: ::std::string::String, + #[doc = "Machine-readable validation error code."] + #[serde(rename = "type")] + pub type_: ::std::string::String, +} +#[doc = " Error types."] +pub mod error { + #[doc = r" Error from a `TryFrom` or `FromStr` implementation."] + pub struct ConversionError(::std::borrow::Cow<'static, str>); + impl ::std::error::Error for ConversionError {} + impl ::std::fmt::Display for ConversionError { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> Result<(), ::std::fmt::Error> { + ::std::fmt::Display::fmt(&self.0, f) + } + } + impl ::std::fmt::Debug for ConversionError { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> Result<(), ::std::fmt::Error> { + ::std::fmt::Debug::fmt(&self.0, f) + } + } + impl From<&'static str> for ConversionError { + fn from(value: &'static str) -> Self { + Self(value.into()) + } + } + impl From for ConversionError { + fn from(value: String) -> Self { + Self(value.into()) + } + } +} diff --git a/crates/moderation/src/typesafe.rs b/crates/moderation/src/typesafe/mod.rs similarity index 65% rename from crates/moderation/src/typesafe.rs rename to crates/moderation/src/typesafe/mod.rs index 7817f17..865cc72 100644 --- a/crates/moderation/src/typesafe.rs +++ b/crates/moderation/src/typesafe/mod.rs @@ -2,10 +2,18 @@ use std::{collections::HashMap, time::Duration}; use anyhow::{Result, anyhow}; use reqwest::header::{AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderValue}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; use tracing::debug; +use crate::typesafe::generated::{ + False, Instructions, NoulCriteria, NoulQuestion, Question, SystemOneRequest, SystemOneResponse, + True, +}; + +#[allow(clippy::all, clippy::pedantic, dead_code)] +mod generated; + +pub use generated::{Answer, State}; + pub(crate) struct Client { base_url: reqwest::Url, client: reqwest::Client, @@ -17,41 +25,6 @@ pub(crate) struct Config { pub timeout: Duration, } -#[derive(Serialize)] -#[serde(tag = "type", rename_all = "lowercase")] -pub(crate) enum Question { - Noul { - instructions: String, - criteria: NoulCriteria, - }, -} - -#[derive(Serialize)] -pub(crate) struct NoulCriteria { - #[serde(rename = "true")] - pub when_true: &'static str, - #[serde(rename = "false")] - pub when_false: &'static str, -} - -#[derive(Deserialize, Debug)] -#[serde(tag = "type", rename_all = "lowercase")] -pub(crate) enum Answer { - Noul { noul: f32 }, -} - -#[derive(Serialize)] -struct Request { - model: &'static str, - state: Value, - questions: HashMap, -} - -#[derive(Deserialize)] -struct Response { - answers: HashMap, -} - impl Client { pub(crate) fn build(conf: &Config) -> Result { let mut auth = HeaderValue::from_str(&format!("Bearer {}", conf.api_key))?; @@ -72,13 +45,14 @@ impl Client { pub(crate) async fn query( &self, - state: Value, + state: State, questions: HashMap, ) -> Result> { - let req = Request { - model: "jev-latest", + let req = SystemOneRequest { + model: String::from("jev-latest"), state, questions, + assets: HashMap::new(), }; let body = serde_json::to_string(&req)?; debug!(%body, "decisions api request"); @@ -98,6 +72,17 @@ impl Client { return Err(anyhow!("decisions api returned {status}: {detail}")); } - Ok(res.json::().await?.answers) + Ok(res.json::().await?.answers) } } + +pub fn noul(instructions: String, when_true: String, when_false: String) -> Question { + Question::NoulQuestion(NoulQuestion { + criteria: Some(NoulCriteria { + true_: Some(True::String(when_true)), + false_: Some(False::String(when_false)), + }), + instructions: Some(Instructions::String(instructions)), + type_: String::from("noul"), + }) +} diff --git a/crates/moderation/typesafe/openapi.json b/crates/moderation/typesafe/openapi.json new file mode 100644 index 0000000..25f234c --- /dev/null +++ b/crates/moderation/typesafe/openapi.json @@ -0,0 +1 @@ +{"openapi":"3.1.0","info":{"title":"TypeSafe","description":"Ask yes/no questions, evaluate statements, select choices, or assign ratings to your content. Send your API key in the Authorization header as `Bearer `. Use GET /v1/models to discover available model names.","version":"0.2.0"},"servers":[{"url":"https://api.typesafe.ai"}],"paths":{"/v1/models":{"get":{"summary":"List models","description":"List the models and aliases available to the authenticated account.\n\nPass a returned model name as `model` in a POST /v1/systemone request.","operationId":"models_v1_v1_models_get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ModelMetadataList"}}}},"400":{"description":"Bad Request","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPBadRequestError"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}},"security":[{"HTTPBearer":[]}]}},"/v1/systemone":{"post":{"summary":"System One","description":"Answer one or more questions about the content supplied in `state`.\n\nYou can mix question types in one request. Answers use the same names as the\nquestions, so you can match each result to its question. The response also includes\nthe model used and token usage.","operationId":"systemone_v1_systemone_post","parameters":[{"name":"TypeSafe-Danger-Zone-Images","in":"header","description":"Set to true to use assets or box questions.","schema":{"type":"string","enum":["true"]},"example":"true"},{"name":"TypeSafe-Danger-Zone-Long-Context","in":"header","description":"Set to true to use long context.","schema":{"type":"string","enum":["true"]},"example":"true"}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/SystemOneRequest"},"examples":{"noul":{"summary":"Noul","value":{"model":"jev-latest","state":"I was charged twice for my subscription this month. Please refund the extra charge.","questions":{"billing":{"type":"noul","instructions":"Is this message about billing?"}}}},"choice":{"summary":"Choice","value":{"model":"jev-latest","state":"I was charged twice for my subscription this month. Please refund the extra charge.","questions":{"tone":{"type":"choice","instructions":"What is the tone of this message?","criteria":{"angry":"An upset or hostile message","calm":"A neutral or polite message","excited":"An enthusiastic or eager message"}}}}},"score":{"summary":"Score","value":{"model":"jev-latest","state":"I was charged twice for my subscription this month. Please refund the extra charge.","questions":{"urgency":{"type":"score","instructions":"How urgent is this message?","criteria":["Can wait","Needs attention this week","Needs attention today"]}}}},"box":{"summary":"Box (requires TypeSafe-Danger-Zone-Images: true)","value":{"model":"jev-preview","state":"photo","questions":{"red_rectangle":{"type":"box","instructions":"Locate the red rectangle."}},"assets":{"photo":{"type":"image","format":"png","data_base64":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAHElEQVR4nGP4TyJgoKcGBgbsaFQDfTUQB4aDBgDAen2fpitsVAAAAABJRU5ErkJggg=="}}}}}},"multipart/form-data":{"schema":{"type":"object","description":"The JSON request as the first part, named request, then each image as its own part. Every image part needs a unique, non-empty name and must be used by exactly one asset's part.","properties":{"request":{"description":"The JSON request, as the first part.","contentMediaType":"application/json","contentSchema":{"$ref":"#/components/schemas/SystemOneRequest"}}},"required":["request"],"additionalProperties":{"type":"string","description":"An image, named by the asset part that uses it.","contentMediaType":"image/*"}},"encoding":{"request":{"contentType":"application/json"}}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SystemOneResponse"},"examples":{"noul":{"summary":"Noul","value":{"model":"jev-latest","answers":{"billing":{"type":"noul","noul":0.98}},"usage":{"input_tokens":120,"output_tokens":12}}},"choice":{"summary":"Choice","value":{"model":"jev-latest","answers":{"tone":{"type":"choice","choice":"calm","confidence":0.7,"probabilities":{"angry":0.15,"calm":0.8,"excited":0.05}}},"usage":{"input_tokens":120,"output_tokens":12}}},"score":{"summary":"Score","value":{"model":"jev-latest","answers":{"urgency":{"type":"score","score":1.7,"confidence":0.55,"legend":{"0":"Can wait","1":"Needs attention this week","2":"Needs attention today"},"probabilities":{"0":0.1,"1":0.1,"2":0.8}}},"usage":{"input_tokens":120,"output_tokens":12}}},"box":{"summary":"Box","value":{"model":"jev-preview","answers":{"red_rectangle":{"type":"box","box":{"top":0.2,"bottom":0.66,"left":0.19,"right":0.68},"confidence":null}},"usage":{"input_tokens":285,"output_tokens":20}}}}}}},"400":{"description":"Bad Request","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPBadRequestError"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}},"security":[{"HTTPBearer":[]}]}}},"components":{"schemas":{"Answer":{"oneOf":[{"$ref":"#/components/schemas/NoulAnswer","title":"Noul"},{"$ref":"#/components/schemas/ChoiceAnswer","title":"Choice"},{"$ref":"#/components/schemas/ScoreAnswer","title":"Score"},{"$ref":"#/components/schemas/BoxAnswer","title":"Box"}],"discriminator":{"propertyName":"type","mapping":{"noul":"#/components/schemas/NoulAnswer","choice":"#/components/schemas/ChoiceAnswer","score":"#/components/schemas/ScoreAnswer","box":"#/components/schemas/BoxAnswer"}},"description":"An answer whose type matches the corresponding question."},"Asset":{"properties":{"type":{"type":"string","const":"image","title":"Type","description":"Identifies an image asset."},"format":{"type":"string","enum":["png","jpg","jpeg"],"title":"Format","description":"The image format, required with data_base64. The actual format is detected from the bytes."},"data_base64":{"type":"string","title":"Data Base64","description":"Base64-encoded image data."},"part":{"type":"string","title":"Part","description":"In a multipart request, the name of the image part this asset uses. With part, data_base64 is ignored."}},"type":"object","title":"Asset","description":"An image that replaces its placeholder: its bytes in data_base64 with their format, or, in a multipart request, the name of the image part that holds them. Requests with images must send the header TypeSafe-Danger-Zone-Images: true."},"BoxAnswer":{"properties":{"type":{"type":"string","const":"box","title":"Type","description":"Identifies a box answer.","examples":["box"]},"box":{"$ref":"#/components/schemas/BoxCoordinates","description":"The box's edges as fractions of image height and width, from 0 to 1. Each coordinate is the expected value of its edge's predicted position.","examples":[{"bottom":0.66,"left":0.19,"right":0.68,"top":0.2}]},"confidence":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Confidence","description":"Confidence for the given box, from 0 to 1, with 1 being more confident. Null when the model does not provide confidence.","examples":[null,0.9]}},"type":"object","required":["type","box","confidence"],"title":"BoxAnswer","description":"A located box with normalized edge coordinates and confidence."},"BoxCoordinates":{"properties":{"top":{"type":"number","title":"Top","description":"Top edge of the box, as a fraction of image height from the top (0-1).","examples":[0.2]},"bottom":{"type":"number","title":"Bottom","description":"Bottom edge of the box, as a fraction of image height from the top (0-1).","examples":[0.66]},"left":{"type":"number","title":"Left","description":"Left edge of the box, as a fraction of image width from the left (0-1).","examples":[0.19]},"right":{"type":"number","title":"Right","description":"Right edge of the box, as a fraction of image width from the left (0-1).","examples":[0.68]}},"type":"object","required":["top","bottom","left","right"],"title":"BoxCoordinates","description":"Normalized box edges. Vertical coordinates are measured from the top of the image; horizontal coordinates are measured from the left."},"BoxQuestion":{"properties":{"type":{"type":"string","const":"box","title":"Type","description":"Identifies a box question."},"instructions":{"anyOf":[{"type":"string"},{"additionalProperties":true,"type":"object"},{"items":{},"type":"array"},{"type":"null"}],"x-accepts":"string | object | array","title":"Instructions","description":"What to locate in the image."}},"type":"object","required":["type"],"title":"BoxQuestion","description":"A question that locates what its instructions describe in the request's image. Requests with box questions must send the header TypeSafe-Danger-Zone-Images: true."},"ChoiceAnswer":{"properties":{"type":{"type":"string","const":"choice","title":"Type","description":"Identifies a selection from the requested choices.","examples":["choice"]},"choice":{"type":"string","title":"Choice","description":"The name of the choice with the highest probability among the question's criteria.","examples":["angry"]},"confidence":{"type":"number","title":"Confidence","description":"Confidence in the selected choice, from 0 to 1. Higher values indicate greater certainty; use lower values to flag uncertain selections for review.","examples":[0.9]},"probabilities":{"additionalProperties":{"type":"number"},"type":"object","title":"Probabilities","description":"Probability of each choice in criteria, keyed by choice name, from 0 to 1. Shows how likely the alternatives are; values sum to approximately 1.","examples":[{"angry":0.8,"calm":0.1,"excited":0.1}]}},"type":"object","required":["type","choice","confidence","probabilities"],"title":"ChoiceAnswer","description":"The selected choice, confidence, and probabilities for a choice question."},"ChoiceQuestion":{"properties":{"type":{"type":"string","const":"choice","title":"Type","description":"Identifies a question that selects one of the choices in criteria.","examples":["choice"]},"instructions":{"anyOf":[{"type":"string"},{"additionalProperties":true,"type":"object"},{"items":{},"type":"array"},{"type":"null"}],"x-accepts":"string | object | array","title":"Instructions","description":"What the model should decide when choosing an option.","examples":["What is the tone of this message?"]},"criteria":{"additionalProperties":{"anyOf":[{"type":"string"},{"additionalProperties":true,"type":"object"},{"items":{},"type":"array"},{"type":"null"}],"x-accepts":"string | object | array"},"type":"object","minProperties":1,"maxProperties":255,"title":"Criteria","description":"Choice names and descriptions of when each applies. A choice without a description is interpreted by its name alone.","examples":[{"angry":"An upset or hostile message","calm":"A neutral or polite message","excited":"An enthusiastic or eager message"}]}},"type":"object","required":["type","criteria"],"title":"ChoiceQuestion","description":"A question that selects one option from the choices you define."},"HTTPBadRequestError":{"type":"object","title":"HTTPBadRequestError","required":["detail"],"description":"Application validation failures returned with HTTP status 400.","properties":{"detail":{"title":"Detail","anyOf":[{"type":"string"},{"type":"object","additionalProperties":true}],"description":"Explanation of the invalid request. String details and message fields are bounded, with shortened text ending with …."}}},"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail","description":"Validation errors describing which request values are missing or invalid.","examples":[[{"loc":["body","state"],"msg":"Field required","type":"missing"}]]}},"type":"object","title":"HTTPValidationError","description":"Request validation failures returned with HTTP status 422."},"ModelMetadata":{"properties":{"name":{"type":"string","title":"Name","description":"Model name or alias accepted by the request's model field.","examples":["jev-latest"]},"description":{"type":"string","title":"Description","description":"Human-readable description of the model and its capabilities.","examples":["General-purpose system one model."]},"release_date":{"type":"string","title":"Release Date","description":"Model release date, formatted as YYYY-MM-DD.","examples":["2026-09-15"]}},"type":"object","required":["name","description","release_date"],"title":"ModelMetadata","description":"A model or model alias available to the authenticated account."},"ModelMetadataList":{"properties":{"models":{"items":{"$ref":"#/components/schemas/ModelMetadata"},"type":"array","title":"Models","description":"Available models and aliases. Use a model's name in POST /v1/systemone requests.","examples":[[{"description":"General-purpose system one model.","name":"jev-latest","release_date":"2026-09-15"}]]}},"type":"object","required":["models"],"title":"ModelMetadataList","description":"Models and aliases available to the authenticated account."},"NoulAnswer":{"properties":{"type":{"type":"string","const":"noul","title":"Type","description":"Identifies a yes/no answer.","examples":["noul"]},"noul":{"type":"number","title":"Noul","description":"Probability of a yes answer or a true statement, from 0 to 1. Values near 1 favor yes or true, values near 0 favor no or false, and values near 0.5 indicate uncertainty.","examples":[0.98]}},"type":"object","required":["type","noul"],"title":"NoulAnswer","description":"The probability of a yes answer or a true statement."},"NoulCriteria":{"properties":{"true":{"anyOf":[{"type":"string"},{"additionalProperties":true,"type":"object"},{"items":{},"type":"array"},{"type":"null"}],"x-accepts":"string | object | array","title":"True","description":"What counts as a yes answer.","examples":["The message is unsolicited advertising."]},"false":{"anyOf":[{"type":"string"},{"additionalProperties":true,"type":"object"},{"items":{},"type":"array"},{"type":"null"}],"x-accepts":"string | object | array","title":"False","description":"What counts as a no answer.","examples":["The message is a legitimate conversation."]}},"type":"object","title":"NoulCriteria","description":"Criteria defining what counts as a yes or no answer."},"NoulQuestion":{"properties":{"type":{"type":"string","const":"noul","title":"Type","description":"Identifies a yes/no question or statement.","examples":["noul"]},"instructions":{"anyOf":[{"type":"string"},{"additionalProperties":true,"type":"object"},{"items":{},"type":"array"},{"type":"null"}],"x-accepts":"string | object | array","title":"Instructions","description":"The yes/no question or statement to evaluate.","examples":["Is this message spam?","This message contains unsolicited advertising.",{"task":"Identify unsolicited advertising."}]},"criteria":{"anyOf":[{"$ref":"#/components/schemas/NoulCriteria"},{"type":"null"}],"description":"Criteria clarifying what counts as a yes or no answer.","examples":[{"false":"A legitimate conversation","true":"Unsolicited advertising"}]}},"type":"object","required":["type"],"title":"NoulQuestion","description":"A yes/no question or statement, answered with the probability of yes or true."},"Question":{"oneOf":[{"$ref":"#/components/schemas/NoulQuestion","title":"Noul"},{"$ref":"#/components/schemas/ChoiceQuestion","title":"Choice"},{"$ref":"#/components/schemas/ScoreQuestion","title":"Score"},{"$ref":"#/components/schemas/BoxQuestion","title":"Box"}],"discriminator":{"propertyName":"type","mapping":{"noul":"#/components/schemas/NoulQuestion","choice":"#/components/schemas/ChoiceQuestion","score":"#/components/schemas/ScoreQuestion","box":"#/components/schemas/BoxQuestion"}},"description":"A question about the supplied content."},"ScoreAnswer":{"properties":{"type":{"type":"string","const":"score","title":"Type","description":"Identifies a rating against the requested score levels.","examples":["score"]},"score":{"type":"number","title":"Score","description":"Expected score: the probability-weighted average of the rubric levels. May fall between integer levels.","examples":[1.7]},"confidence":{"type":"number","title":"Confidence","description":"Confidence in the score, from 0 to 1. Higher values indicate greater certainty; use lower values to flag uncertain ratings for review.","examples":[0.9]},"legend":{"additionalProperties":{"anyOf":[{"type":"string"},{"additionalProperties":true,"type":"object"},{"items":{},"type":"array"}],"x-accepts":"string | object | array"},"type":"object","title":"Legend","description":"The requested criteria mapped to their score levels, so you can interpret the score.","examples":[{"0":"Can wait","1":"Needs attention this week","2":"Needs attention today"}]},"probabilities":{"additionalProperties":{"type":"number"},"type":"object","title":"Probabilities","description":"Probability of each score level, from 0 to 1, using the same keys as legend. Shows how likely the alternatives are; values sum to approximately 1.","examples":[{"0":0.1,"1":0.1,"2":0.8}]}},"type":"object","required":["type","score","confidence","legend","probabilities"],"title":"ScoreAnswer","description":"An expected score with its rubric, confidence, and score-level probabilities."},"ScoreQuestion":{"properties":{"type":{"type":"string","const":"score","title":"Type","description":"Identifies a question that rates the content using the levels in criteria.","examples":["score"]},"instructions":{"anyOf":[{"type":"string"},{"additionalProperties":true,"type":"object"},{"items":{},"type":"array"},{"type":"null"}],"x-accepts":"string | object | array","title":"Instructions","description":"What the model should rate.","examples":["How urgent is this message?"]},"criteria":{"items":{"anyOf":[{"type":"string"},{"additionalProperties":true,"type":"object"},{"items":{},"type":"array"}],"x-accepts":"string | object | array"},"type":"array","minItems":1,"maxItems":10,"title":"Criteria","description":"Ordered descriptions of the score levels. Each description's position determines its score, starting at zero.","examples":[["Can wait","Needs attention this week","Needs attention today"]]}},"type":"object","required":["type","criteria"],"title":"ScoreQuestion","description":"A question that assigns a score using an ordered rubric."},"SystemOneRequest":{"properties":{"model":{"type":"string","title":"Model","description":"Name or alias of the model to use. Available names are returned by GET /v1/models.","examples":["jev-latest"]},"state":{"anyOf":[{"type":"string"},{"additionalProperties":true,"type":"object"},{"items":{},"type":"array"}],"x-accepts":"string | object | array","title":"State","description":"The content all questions in this request refer to.","examples":["I was charged twice. Please help.",{"message":"Please help.","subject":"Duplicate charge"}]},"questions":{"additionalProperties":{"$ref":"#/components/schemas/Question"},"type":"object","minProperties":1,"title":"Questions","description":"Questions to ask about the content, each with a name you choose. The response uses those names to identify the answers.","examples":[{"billing":{"instructions":"Is this message about billing?","type":"noul"}}]},"assets":{"additionalProperties":{"$ref":"#/components/schemas/Asset"},"type":"object","minProperties":1,"title":"Assets","description":"Media assets keyed by an opaque placeholder (recommend a per-image UUID). Each placeholder, wherever it appears as or inside a string in state, question instructions, or criteria, is replaced with the asset. Image assets require a vision-language model."}},"type":"object","required":["model","state","questions"],"title":"SystemOneRequest","description":"Content and named questions to evaluate together using a TypeSafe model."},"SystemOneResponse":{"properties":{"model":{"type":"string","title":"Model","description":"Name of the model that answered the questions. May differ from the alias supplied in the request.","examples":["jev-latest"]},"answers":{"additionalProperties":{"$ref":"#/components/schemas/Answer"},"type":"object","minProperties":1,"title":"Answers","description":"Answers keyed by the question names supplied in the request. Each answer's type matches its question's type.","examples":[{"billing":{"noul":0.98,"type":"noul"}}]},"usage":{"$ref":"#/components/schemas/Usage","description":"Input and output token counts for this evaluation.","examples":[{"input_tokens":120,"output_tokens":12}]}},"type":"object","required":["model","answers","usage"],"title":"SystemOneResponse","description":"Answers grouped by question name, with the model used and token usage."},"Usage":{"properties":{"input_tokens":{"type":"integer","title":"Input Tokens","description":"Number of billable input tokens used to evaluate the request.","examples":[120]},"output_tokens":{"type":"integer","title":"Output Tokens","description":"Number of output tokens used to answer the questions. Output tokens are currently free of charge.","examples":[12]}},"type":"object","required":["input_tokens","output_tokens"],"title":"Usage","description":"Token usage for the request."},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location","description":"Path to the invalid value: the request location followed by field names and array indices. May be empty.","examples":[["body","questions","urgency","score","criteria"]]},"msg":{"type":"string","title":"Message","description":"Human-readable explanation of the validation failure. May be shortened with ….","examples":["Field required"]},"type":{"type":"string","title":"Error Type","description":"Machine-readable validation error code.","examples":["missing"]},"input":{"type":"string","title":"Input","description":"Text preview of the rejected input. May be shortened with ….","examples":["{\"type\":\"score\"}"]},"ctx":{"type":"object","title":"Context","description":"Additional context used to explain the validation failure. Entries may be omitted or shortened.","examples":[{"min_length":1}]}},"type":"object","required":["loc","msg","type"],"title":"ValidationError","description":"A request validation error at a specific field or array element."}},"securitySchemes":{"HTTPBearer":{"type":"http","scheme":"bearer"}}}} \ No newline at end of file diff --git a/mise.toml b/mise.toml index f2b671a..c3e2864 100644 --- a/mise.toml +++ b/mise.toml @@ -6,6 +6,8 @@ node = "26.12.1" pnpm = "12.14.0" # Keep in sync with the sqlx version in crates/database. "cargo:sqlx-cli" = { version = "0.9.0", default-features = false, features = "postgres,rustls,sqlx-toml" } +"cargo:cargo-typify" = "0.8.0" +jq = "1.8.2" [tasks."db:migrate"] description = "Apply pending Postgres migrations" @@ -26,3 +28,13 @@ run = "cargo sqlx prepare --config sqlx.toml" description = "Generate the frontend's Kysely types from the migrated Postgres schema" dir = "{{config_root}}/web" run = "pnpm db:codegen" + +[tasks."moderation:codegen"] +description = "Generate TypeSafe API types from the vendored OpenAPI spec" +dir = "{{config_root}}/crates/moderation" +run = """ +jq '{"$schema": "https://json-schema.org/draft/2020-12/schema", definitions: .components.schemas}' typesafe/openapi.json \ + | sed 's#/components/schemas/#/definitions/#g' > target-schemas.json +cargo typify target-schemas.json --output src/typesafe/generated.rs --no-builder +rm target-schemas.json +""" From 27136a6491a419724af7375c5b5afe02243f666b Mon Sep 17 00:00:00 2001 From: Kshitij Chauhan Date: Fri, 6 Nov 2026 22:04:48 +0000 Subject: [PATCH 5/5] Parse systemone url correctly --- crates/moderation/src/typesafe/mod.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/moderation/src/typesafe/mod.rs b/crates/moderation/src/typesafe/mod.rs index 865cc72..f135570 100644 --- a/crates/moderation/src/typesafe/mod.rs +++ b/crates/moderation/src/typesafe/mod.rs @@ -15,7 +15,7 @@ mod generated; pub use generated::{Answer, State}; pub(crate) struct Client { - base_url: reqwest::Url, + system_one_url: reqwest::Url, client: reqwest::Client, } @@ -37,8 +37,11 @@ impl Client { .default_headers(headers) .build()?; + let base_url: reqwest::Url = conf.base_url.parse()?; + let system_one_url = base_url.join("v1/systemone")?; + Ok(Self { - base_url: conf.base_url.parse()?, + system_one_url, client, }) } @@ -57,10 +60,9 @@ impl Client { let body = serde_json::to_string(&req)?; debug!(%body, "decisions api request"); - let url = self.base_url.join("/v1/systemone")?; let res = self .client - .post(url) + .post(self.system_one_url.clone()) .header(CONTENT_TYPE, "application/json") .body(body) .send()