Add structured moderation assessments with revision history

This commit is contained in:
2026-10-17 19:47:28 -04:00
parent aca728b557
commit 428aefdf24
18 changed files with 1182 additions and 2 deletions
+46
View File
@@ -41,6 +41,49 @@ export function isModerationCaseOpen(status: ModerationCaseStatus): boolean {
return status === 'open' || status === 'in_review' || status === 'escalated';
}
export const moderationConcernCategories = [
'hate_speech',
'harassment',
'threats',
'safeguarding',
'spam',
'cheating',
'other'
] as const;
export type ModerationConcernCategory = (typeof moderationConcernCategories)[number];
export const moderationConcernLabels: Record<ModerationConcernCategory, string> = {
hate_speech: 'Hate speech',
harassment: 'Harassment',
threats: 'Threats',
safeguarding: 'Safeguarding',
spam: 'Spam',
cheating: 'Cheating',
other: 'Other'
};
export const moderationFindingOutcomes = [
'substantiated',
'inconclusive',
'unsubstantiated'
] as const;
export type ModerationFindingOutcome = (typeof moderationFindingOutcomes)[number];
export const moderationFindingLabels: Record<ModerationFindingOutcome, string> = {
substantiated: 'Substantiated',
inconclusive: 'Inconclusive',
unsubstantiated: 'Unsubstantiated'
};
export interface ModerationCaseAssessment {
revision: number;
summary: string;
rules: string[];
findings: {
category: ModerationConcernCategory;
finding: ModerationFindingOutcome;
detail: string;
}[];
actor: string;
createdAt: string;
}
export interface ModerationPlayer {
game: ModerationGame;
guid: string;
@@ -112,6 +155,7 @@ export interface ModerationCase {
createdAt: string;
updatedAt: string;
revision: number;
assessment?: ModerationCaseAssessment | null;
evidence: {
id: string;
text: string;
@@ -127,6 +171,8 @@ export interface ModerationCaseDetail {
sanctions: ModerationSanction[];
linkedActions?: ModerationCaseAction[];
availableActions?: ModerationCaseAction[];
assessment?: ModerationCaseAssessment | null;
assessmentHistory?: ModerationCaseAssessment[];
events: ModerationEvent[];
coverage: ModerationCoverage[];
assignees: { id: string; username: string }[];
+7 -1
View File
@@ -31,7 +31,8 @@ import { migrateTemperatures } from '../hosts/temperatures.js';
import {
migrateModeration,
migrateModerationCases,
migrateModerationCaseActions
migrateModerationCaseActions,
migrateModerationCaseAssessments
} from '../moderation/schema.js';
import type { Migration } from './migration-runner.js';
@@ -179,6 +180,11 @@ export const migrations: readonly Migration[] = [
version: 31,
name: 'moderation case links to existing actions',
run: migrateModerationCaseActions
},
{
version: 32,
name: 'moderation case assessment revision history',
run: migrateModerationCaseAssessments
}
];
+26
View File
@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { archivedEvidenceSnapshots } from './evidence.js';
import { caseActionReference, recordedCaseAction, isCaseActionLinked } from './case-actions.js';
import { assessmentDraft } from './assessments.js';
import { AuthError } from '../auth/index.js';
import { moderatorMaximumRestrictionMs, type AuthUser } from '../../auth.js';
import {
@@ -472,6 +473,31 @@ export function mutateModeration(actorId: string, input: Record<string, unknown>
if (input.op === 'action') return liveAction(actor, input, now);
if (input.op === 'create-case') return createCase(actor, input, now);
if (input.op === 'update-case') return updateCase(actor, input, now);
if (input.op === 'assess-case') {
const row = caseRow(input.id);
checkCaseRevision(row, input.revision);
const draft = assessmentDraft(input);
const data = JSON.stringify(draft);
const prior = db()
.prepare(
'SELECT data FROM moderation_case_assessments WHERE case_id=? ORDER BY revision DESC LIMIT 1'
)
.get(row.id) as { data: string } | undefined;
if (prior?.data === data) throw new AuthError('Change the assessment before saving.');
db()
.prepare('INSERT INTO moderation_case_assessments VALUES(?,?,?,?,?,?)')
.run(row.id, row.revision + 1, data, actor.id, actor.username, now);
touchCase(row.id, now);
moderationEvent(
actor,
'case.assessed',
row,
row.id,
JSON.stringify({ assessmentRevision: row.revision + 1, summary: draft.summary }),
now
);
return { id: row.id };
}
if (input.op === 'link-case-action') {
const row = caseRow(input.id);
checkCaseRevision(row, input.revision);
+63
View File
@@ -0,0 +1,63 @@
import {
moderationConcernCategories,
moderationFindingOutcomes,
type ModerationCaseAssessment
} from '../../moderation.js';
import { AuthError } from '../auth/index.js';
import { db } from '../db/index.js';
import { textField } from './core.js';
type AssessmentDraft = Pick<ModerationCaseAssessment, 'summary' | 'rules' | 'findings'>;
export function assessmentDraft(input: Record<string, unknown>): AssessmentDraft {
const summary = textField(input.summary, 'Assessment summary', 2000);
if (!Array.isArray(input.rules) || input.rules.length > 20)
throw new AuthError('Choose up to 20 applicable rules.');
const rules = input.rules.map((rule) => textField(rule, 'Applicable rule', 160));
if (
!Array.isArray(input.findings) ||
!input.findings.length ||
input.findings.length > moderationConcernCategories.length
)
throw new AuthError('Record between 1 and 7 findings, one per concern category.');
const categories = new Set<string>();
const findings = input.findings.map((value): AssessmentDraft['findings'][number] => {
if (!value || typeof value !== 'object')
throw new AuthError('Choose a concern category and finding.');
const category = moderationConcernCategories.find(
(category) => category === value.category
);
const finding = moderationFindingOutcomes.find((finding) => finding === value.finding);
if (!category || !finding)
throw new AuthError('Choose a known concern category and finding.');
if (categories.has(category))
throw new AuthError('Record each concern category only once.');
categories.add(category);
return { category, finding, detail: textField(value.detail, 'Finding detail', 1000) };
});
return { summary, rules, findings };
}
export function caseAssessmentHistory(caseId: string): ModerationCaseAssessment[] {
return (
db()
.prepare(
`SELECT revision,data,actor,created_at AS createdAt
FROM moderation_case_assessments WHERE case_id=? ORDER BY revision DESC`
)
.all(caseId) as { revision: number; data: string; actor: string; createdAt: string }[]
).map(({ data, ...revision }) => ({ ...JSON.parse(data), ...revision }));
}
export function currentCaseAssessment(caseId: string): ModerationCaseAssessment | null {
const row = db()
.prepare(
`SELECT revision,data,actor,created_at AS createdAt
FROM moderation_case_assessments WHERE case_id=? ORDER BY revision DESC LIMIT 1`
)
.get(caseId) as
{ revision: number; data: string; actor: string; createdAt: string } | undefined;
if (!row) return null;
const { data, ...revision } = row;
return { ...JSON.parse(data), ...revision };
}
+4
View File
@@ -3,6 +3,7 @@ import { AuthError } from '../auth/index.js';
import { moderationActor, textField } from './core.js';
import { moderationOverview } from './overview.js';
import { caseActionLists } from './case-actions.js';
import { caseAssessmentHistory } from './assessments.js';
import type {
ModerationCase,
ModerationCaseDetail,
@@ -68,6 +69,7 @@ export function moderationCaseDetail(actorId: string, id: string): ModerationCas
AND disabled=0 AND must_change_password=0 ORDER BY username COLLATE NOCASE,id`
)
.all() as ModerationCaseDetail['assignees'];
const assessmentHistory = caseAssessmentHistory(caseId);
return {
case: { ...row, evidence },
player: overview.players[0] ?? {
@@ -82,6 +84,8 @@ export function moderationCaseDetail(actorId: string, id: string): ModerationCas
},
sanctions,
...caseActionLists(caseId, row, overview.checkedAt),
assessment: assessmentHistory[0] ?? null,
assessmentHistory,
events,
coverage: overview.coverage,
assignees,
+2
View File
@@ -1,4 +1,5 @@
import { db } from '../db/index.js';
import { currentCaseAssessment } from './assessments.js';
import { moderationActor, moderationNow, policyRevision } from './core.js';
import { runtimeFreshMs } from './actions.js';
import { refreshActionStates } from './runtime.js';
@@ -200,6 +201,7 @@ export function moderationOverview(actorId: string, query: ModerationQuery): Mod
"CASE WHEN p.status IN ('open','in_review','escalated') THEN 0 ELSE 1 END,CASE p.priority WHEN 'urgent' THEN 0 WHEN 'high' THEN 1 WHEN 'normal' THEN 2 ELSE 3 END,p.updated_at DESC,p.id"
).map((c) => ({
...c,
assessment: currentCaseAssessment(c.id),
evidence: handle
.prepare(
'SELECT id,text,source,created_at AS createdAt,actor FROM moderation_evidence WHERE case_id=? ORDER BY created_at,id'
+14
View File
@@ -165,3 +165,17 @@ export function migrateModerationCaseActions(handle: Database.Database): void {
ON moderation_case_action_links(action_type,action_id);
`);
}
export function migrateModerationCaseAssessments(handle: Database.Database): void {
handle.exec(`
CREATE TABLE IF NOT EXISTS moderation_case_assessments (
case_id TEXT NOT NULL REFERENCES moderation_cases(id),
revision INTEGER NOT NULL CHECK(revision > 0),
data TEXT NOT NULL,
actor_id TEXT NOT NULL,
actor TEXT NOT NULL,
created_at TEXT NOT NULL,
PRIMARY KEY(case_id,revision)
);
`);
}
@@ -0,0 +1,167 @@
<script lang="ts">
import {
moderationConcernLabels,
moderationFindingLabels,
type ModerationCaseAssessment
} from '$lib/moderation.js';
import Button from '../primitives/Button.svelte';
import Badge from '../primitives/Badge.svelte';
import RelativeTime from '../RelativeTime.svelte';
import CaseAssessmentDialog from './CaseAssessmentDialog.svelte';
let {
caseId,
revision,
assessment = null,
history = [],
permitted,
onsubmitted
}: {
caseId: string;
revision: number;
assessment?: ModerationCaseAssessment | null;
history?: ModerationCaseAssessment[];
permitted: boolean;
onsubmitted: () => void;
} = $props();
let open = $state(false);
const previous = $derived(history.filter((item) => item.revision !== assessment?.revision));
$effect(() => {
void caseId;
open = false;
});
</script>
{#snippet assessmentContent(value: ModerationCaseAssessment)}
<p class="assessment-summary">{value.summary}</p>
<div>
<h3>Applicable rules</h3>
{#if value.rules.length}<ul class="rules">
{#each value.rules as rule}<li>{rule}</li>{/each}
</ul>{:else}<p class="hint">No rule references recorded.</p>{/if}
</div>
<ol class="findings" aria-label="Structured findings">
{#each value.findings as finding}<li>
<div class="heading">
<strong>{moderationConcernLabels[finding.category]}</strong><Badge
variant={finding.finding === 'substantiated'
? 'danger'
: finding.finding === 'inconclusive'
? 'warning'
: 'neutral'}>{moderationFindingLabels[finding.finding]}</Badge
>
</div>
<p>{finding.detail}</p>
</li>{/each}
</ol>
<p class="hint">
Revision {value.revision} · {value.actor} · <RelativeTime value={value.createdAt} />
</p>
{/snippet}
<section class="assessment" aria-label="Case assessment">
<div class="heading">
<h2>Case assessment</h2>
{#if permitted}<Button size="sm" onclick={() => (open = true)}
>{assessment ? 'Edit assessment' : 'Assess case'}</Button
>{/if}
</div>
{#if assessment}{@render assessmentContent(assessment)}{:else}<p class="hint">
Not assessed. Record concerns, applicable rules, and what the evidence supports.
</p>{/if}
{#if previous.length}<details>
<summary>Previous assessments ({previous.length})</summary>
<ol class="history">
{#each previous as earlier (earlier.revision)}<li>
{@render assessmentContent(earlier)}
</li>{/each}
</ol>
</details>{/if}
</section>
{#if permitted && open}<CaseAssessmentDialog
bind:open
{caseId}
{revision}
{assessment}
{onsubmitted}
/>{/if}
<style>
.assessment {
display: grid;
gap: var(--space-4);
padding: var(--space-5);
border: 1px solid var(--border-subtle);
border-radius: var(--radius-lg);
background: var(--surface);
min-width: 0;
}
.heading {
display: flex;
align-items: center;
justify-content: space-between;
gap: var(--space-3);
flex-wrap: wrap;
}
h2,
h3,
p {
margin: 0;
}
h2 {
font-size: var(--text-lg);
}
h3 {
font-size: var(--text-sm);
}
p,
.rules {
line-height: var(--leading-body);
overflow-wrap: anywhere;
}
.assessment-summary,
.findings p {
white-space: pre-wrap;
}
.rules {
padding-left: var(--space-5);
margin: var(--space-2) 0 0;
}
.findings,
.history {
list-style: none;
margin: 0;
padding: 0;
display: grid;
gap: var(--space-3);
}
.findings > li {
display: grid;
gap: var(--space-2);
padding: var(--space-3);
background: var(--surface-sunken);
border-radius: var(--radius-md);
}
.hint {
color: var(--text-secondary);
font-size: var(--text-sm);
}
details {
border-top: 1px solid var(--border-subtle);
padding-top: var(--space-3);
}
summary {
cursor: pointer;
font-size: var(--text-sm);
}
.history {
margin-top: var(--space-4);
}
.history > li {
display: grid;
gap: var(--space-3);
padding-bottom: var(--space-4);
border-bottom: 1px solid var(--border-subtle);
}
</style>
@@ -0,0 +1,260 @@
<script lang="ts">
import { untrack } from 'svelte';
import {
moderationConcernCategories,
moderationConcernLabels,
moderationFindingOutcomes,
moderationFindingLabels,
type ModerationCaseAssessment,
type ModerationConcernCategory,
type ModerationFindingOutcome
} from '$lib/moderation.js';
import Dialog from '../primitives/Dialog.svelte';
import Field from '../primitives/Field.svelte';
import Select from '../primitives/Select.svelte';
import Textarea from '../primitives/Textarea.svelte';
import Button from '../primitives/Button.svelte';
interface DraftFinding {
category: ModerationConcernCategory | '';
finding: ModerationFindingOutcome | '';
detail: string;
key: number;
}
let {
open = $bindable(false),
caseId,
revision,
assessment,
onsubmitted
}: {
open?: boolean;
caseId: string;
revision: number;
assessment: ModerationCaseAssessment | null;
onsubmitted: () => void;
} = $props();
const openedRevision = untrack(() => revision);
let summary = $state(untrack(() => assessment?.summary ?? ''));
let rulesText = $state(untrack(() => assessment?.rules.join('\n') ?? ''));
let nextKey = 0;
let findings = $state<DraftFinding[]>(
untrack(
() =>
assessment?.findings.map((finding) => ({ ...finding, key: nextKey++ })) ?? [
{ category: '', finding: '', detail: '', key: nextKey++ }
]
)
);
let busy = $state(false);
let uncertain = $state(false);
let error = $state('');
let pending: Record<string, unknown> | null = null;
async function submit(event: SubmitEvent) {
event.preventDefault();
if (busy) return;
if (!pending) {
const rules = rulesText
.split('\n')
.map((rule) => rule.trim())
.filter(Boolean);
if (
!summary.trim() ||
!findings.length ||
findings.some((item) => !item.category || !item.finding || !item.detail.trim())
) {
error = 'Add a summary and complete each concern, finding, and explanation.';
return;
}
if (rules.length > 20 || rules.some((rule) => rule.length > 160)) {
error = 'Use up to 20 applicable rules, with no more than 160 characters per line.';
return;
}
pending = {
op: 'assess-case',
id: caseId,
revision: openedRevision,
summary: summary.trim(),
rules,
findings: findings.map(({ category, finding, detail }) => ({
category,
finding,
detail: detail.trim()
})),
idempotencyKey: crypto.randomUUID()
};
}
busy = true;
error = '';
try {
const response = await fetch('/api/moderation', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(pending)
});
const result = await response.json().catch(() => null);
if (!response.ok || !result?.ok) {
uncertain = response.status >= 500 || response.ok;
if (!uncertain) pending = null;
throw new Error(result?.error || 'Could not save the assessment.');
}
open = false;
onsubmitted();
} catch (cause) {
if (pending) uncertain = true;
error = cause instanceof Error ? cause.message : 'Could not save the assessment.';
} finally {
busy = false;
}
}
</script>
<Dialog
bind:open
title={assessment ? 'Edit case assessment' : 'Assess case'}
description="Record the current evidence assessment. Each saved revision is preserved in the case history."
{busy}
wide
>
<form onsubmit={submit}>
<fieldset disabled={busy || uncertain}>
<Field label="Current summary" required
><Textarea bind:value={summary} required rows={3} maxlength={2000} /></Field
>
<Field
label="Applicable rules"
hint="One rule or policy reference per line. Up to 20 lines, 160 characters each."
><Textarea bind:value={rulesText} rows={3} maxlength={3220} /></Field
>
<div class="findings">
{#each findings as item, index (item.key)}
<section class="finding" aria-label={`Finding ${index + 1}`}>
<div class="finding-heading">
<h3>Finding {index + 1}</h3>
<Button
size="sm"
disabled={findings.length === 1}
ariaLabel={`Remove finding ${index + 1}`}
onclick={() =>
(findings = findings.filter(
(finding) => finding.key !== item.key
))}>Remove</Button
>
</div>
<div class="paired">
<Field label={`Concern ${index + 1}`} required
><Select
bind:value={item.category}
required
options={[
{ value: '', label: 'Select a concern' },
...moderationConcernCategories.map((category) => ({
value: category,
label: moderationConcernLabels[category],
disabled: findings.some(
(other) =>
other.key !== item.key &&
other.category === category
)
}))
]}
/></Field
>
<Field label={`Finding ${index + 1}`} required
><Select
bind:value={item.finding}
required
options={[
{ value: '', label: 'Select a finding' },
...moderationFindingOutcomes.map((finding) => ({
value: finding,
label: moderationFindingLabels[finding]
}))
]}
/></Field
>
</div>
<Field
label={`Explanation ${index + 1}`}
required
hint="Explain how the evidence supports this finding, including uncertainty."
><Textarea
bind:value={item.detail}
required
rows={3}
maxlength={1000}
/></Field
>
</section>
{/each}
<Button
size="sm"
disabled={findings.length >= moderationConcernCategories.length}
onclick={() =>
(findings = [
...findings,
{ category: '', finding: '', detail: '', key: nextKey++ }
])}>Add finding</Button
>
</div>
</fieldset>
{#if error}<p class="ui-error" role="alert">{error}</p>{/if}
{#if uncertain}<p class="hint">
The result is uncertain. Retry sends the same assessment safely.
</p>{/if}
<div class="actions">
<Button disabled={busy} onclick={() => (open = false)}>Cancel</Button><Button
type="submit"
variant="primary"
loading={busy}>{uncertain ? 'Retry same request' : 'Save assessment'}</Button
>
</div>
</form>
</Dialog>
<style>
form,
fieldset,
.findings,
.finding {
display: grid;
gap: var(--space-4);
min-width: 0;
}
fieldset {
margin: 0;
padding: 0;
border: 0;
}
.finding {
padding: var(--space-4);
border: 1px solid var(--border-subtle);
border-radius: var(--radius-md);
}
.finding-heading,
.actions {
display: flex;
align-items: center;
justify-content: space-between;
flex-wrap: wrap;
gap: var(--space-3);
}
h3 {
margin: 0;
font-size: var(--text-base);
}
.paired {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: var(--space-3);
}
.actions {
justify-content: flex-end;
}
.hint {
font-size: var(--text-sm);
line-height: var(--leading-body);
color: var(--text-secondary);
}
</style>
+4
View File
@@ -68,6 +68,8 @@ export const moderationActionLabels: Record<string, string> = {
'case.appeal_denied': 'Appeal denied',
'case.reopened': 'Case reopened',
'case.updated': 'Case updated',
'case.assessed': 'Assessment revised',
'assess-case': 'Assessment revised',
'case.evidence_attached': 'Evidence attached',
'case.restriction': 'Restriction linked to case',
'case.action_linked': 'Existing action linked to case',
@@ -96,6 +98,8 @@ export function moderationEventDetail(detail: string, action: string): string {
? moderationCaseStatusLabels[item as ModerationCaseStatus]
: item
: '';
if (action === 'case.assessed' && typeof value.summary === 'string')
return `Assessment revision ${value.assessmentRevision}: ${value.summary}`;
if (action === 'case.updated') {
const changes = ['title', 'priority', 'assignee', 'status'].flatMap((key) => {
const change = value[key];
+1 -1
View File
@@ -19,7 +19,7 @@ export const GET: RequestHandler = ({ locals, url }) => {
export const POST: RequestHandler = async ({ locals, request }) => {
try {
const actor = requireUser(locals.user);
const body = await authBody(request, 16 * 1024);
const body = await authBody(request, 64 * 1024);
if (body.op === 'configure')
return json(
moderationMutation(actor.id, body, () =>
+26
View File
@@ -7,6 +7,8 @@
canRevokeModerationRestriction,
moderationCaseStatuses,
moderationCaseStatusLabels,
moderationConcernLabels,
moderationFindingLabels,
type ModerationOverview,
type ModerationQuery,
type ModerationGame
@@ -29,6 +31,7 @@
import Button from '$ui/components/primitives/Button.svelte';
import Tabs from '$ui/components/primitives/Tabs.svelte';
import StatusBadge from '$ui/components/primitives/StatusBadge.svelte';
import Badge from '$ui/components/primitives/Badge.svelte';
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
import RelativeTime from '$ui/components/RelativeTime.svelte';
import Freshness from '$ui/components/Freshness.svelte';
@@ -609,6 +612,22 @@
>
· {moderationGameName(item.game)}
</p>
{#if item.assessment}
<p class="assessment-summary">{item.assessment.summary}</p>
<div class="record-heading" aria-label="Assessment findings">
{#each item.assessment.findings as finding}
<Badge
variant={finding.finding === 'substantiated'
? 'warning'
: 'neutral'}
>
{moderationConcernLabels[finding.category]} · {moderationFindingLabels[
finding.finding
]}
</Badge>
{/each}
</div>
{:else}<p class="hint">Not assessed</p>{/if}
<dl>
<div>
<dt>Priority</dt>
@@ -874,4 +893,11 @@
flex: 1 1 100%;
}
}
.assessment-summary {
display: -webkit-box;
-webkit-box-orient: vertical;
-webkit-line-clamp: 3;
line-clamp: 3;
overflow: hidden;
}
</style>
@@ -31,6 +31,7 @@
import ModerationDialog from '$ui/components/moderation/ModerationDialog.svelte';
import EvidenceMessages from '$ui/components/moderation/EvidenceMessages.svelte';
import LinkCaseActionDialog from '$ui/components/moderation/LinkCaseActionDialog.svelte';
import CaseAssessment from '$ui/components/moderation/CaseAssessment.svelte';
import { preservedConversation } from '$ui/moderation-evidence.js';
let { data }: { data: { moderationCase: ModerationCaseDetail } } = $props();
@@ -159,6 +160,11 @@
cancelRefresh();
void refresh();
}
function assessmentSaved() {
notice = 'Case assessment saved. The earlier assessment remains in revision history.';
cancelRefresh();
void refresh();
}
function archiveContext(raw: string | null | undefined) {
if (!raw) return null;
@@ -273,6 +279,14 @@
<div class="workspace">
<div class="case-main">
<CaseAssessment
caseId={item.id}
revision={item.revision}
assessment={report.assessment}
history={report.assessmentHistory}
{permitted}
onsubmitted={assessmentSaved}
/>
<section class="panel" aria-labelledby="evidence-title">
<div class="section-heading">
<h2 id="evidence-title">