COOKBOOK:_CONTENT_MODERATION_AGENT

import { Tool } from '@akios/core'
import { z } from 'zod'

// Content analysis tool
const analyzeContent = new Tool({
  name: 'analyze_content',
  description: 'Analyze text content for toxicity, spam, and policy violations',
  schema: z.object({
    content: z.string(),
    context: z.string().optional(),
    strictness: z.enum(['low', 'medium', 'high']).default('medium')
  }),
  handler: async ({ content, context, strictness }) => {
    // In practice, use services like Perspective API, OpenAI Moderation, etc.
    const analysis = {
      toxicity_score: calculateToxicity(content),
      spam_score: calculateSpam(content),
      policy_violations: detectPolicyViolations(content),
      categories: categorizeContent(content),
      recommended_action: determineAction(content, strictness)
    }

    return JSON.stringify(analysis)
  }
})

// Helper functions
function calculateToxicity(text: string): number {
  const toxicWords = ['hate', 'stupid', 'idiot', 'worst', 'terrible', 'awful']
  const words = text.toLowerCase().split(' ')
  const toxicCount = words.filter(word => toxicWords.includes(word)).length
  return Math.min(toxicCount * 0.2, 1.0)
}

function calculateSpam(text: string): number {
  // Simple spam detection
  const spamIndicators = [
    text.length < 10, // Too short
    text.includes('http'), // Links
    text.toUpperCase() === text && text.length > 20, // All caps
    /([a-zA-Z])\1{3,}/.test(text), // Repeated characters
  ]
  return spamIndicators.filter(Boolean).length * 0.25
}

function detectPolicyViolations(text: string): string[] {
  const violations = []
  const lowerText = text.toLowerCase()

  if (lowerText.includes('contact info') || /\d{3}-\d{3}-\d{4}/.test(text)) {
    violations.push('personal_information')
  }

  if (lowerText.includes('political') && lowerText.includes('extreme')) {
    violations.push('hate_speech')
  }

  if (lowerText.includes('advertisement') || lowerText.includes('buy now')) {
    violations.push('spam')
  }

  return violations
}

function categorizeContent(text: string): string[] {
  const categories = []
  const lowerText = text.toLowerCase()

  if (lowerText.includes('question') || lowerText.includes('help') || lowerText.includes('?')) {
    categories.push('question')
  }

  if (lowerText.includes('thank') || lowerText.includes('appreciate')) {
    categories.push('positive_feedback')
  }

  if (lowerText.includes('bug') || lowerText.includes('error') || lowerText.includes('broken')) {
    categories.push('bug_report')
  }

  return categories
}

function determineAction(text: string, strictness: string): string {
  const toxicity = calculateToxicity(text)
  const spam = calculateSpam(text)
  const violations = detectPolicyViolations(text)

  const thresholds = {
    low: { toxicity: 0.8, spam: 0.7 },
    medium: { toxicity: 0.6, spam: 0.5 },
    high: { toxicity: 0.4, spam: 0.3 }
  }

  const threshold = thresholds[strictness as keyof typeof thresholds]

  if (toxicity > threshold.toxicity || violations.length > 0) {
    return 'reject'
  }

  if (spam > threshold.spam) {
    return 'flag_for_review'
  }

  return 'approve'
}

import { Agent } from '@akios/core'

const contentModerationAgent = new Agent({
  name: 'ContentModerationAgent',
  model: 'gpt-4',
  systemPrompt: `You are a content moderation specialist. Your responsibilities:

1. Analyze content for safety and appropriateness
2. Detect spam, toxicity, and policy violations
3. Make fair and consistent moderation decisions
4. Provide clear feedback when content is rejected
5. Consider context and intent when evaluating content

Guidelines:
- Be fair but firm - protect community safety
- Give benefit of doubt when intent is ambiguous
- Explain rejections clearly so users can improve
- Consider cultural context and language differences
- Escalate edge cases to human moderators`,
  tools: [analyzeContent],
  guardrails: [
    new ContentSafetyGuardrail({ strictness: 'high' }),
    new ProfanityFilterGuardrail()
  ]
})

// Moderation workflow
async function moderateContent(content: string, context?: string) {
  // Step 1: Analyze content
  const analysis = await analyzeContent.execute({
    content,
    context,
    strictness: 'medium'
  })

  const parsed = JSON.parse(analysis)

  // Step 2: Make decision
  switch (parsed.recommended_action) {
    case 'approve':
      return {
        action: 'approve',
        reason: 'Content meets community guidelines'
      }

    case 'reject':
      return {
        action: 'reject',
        reason: generateRejectionReason(parsed),
        categories: parsed.categories
      }

    case 'flag_for_review':
      // Escalate to human moderator
      await escalateToHuman(content, parsed)
      return {
        action: 'escalated',
        reason: 'Content flagged for human review'
      }

    default:
      return {
        action: 'escalated',
        reason: 'Unable to determine action automatically'
      }
  }
}

function generateRejectionReason(analysis: any): string {
  const reasons = []

  if (analysis.toxicity_score > 0.6) {
    reasons.push('Content appears to contain toxic or harmful language')
  }

  if (analysis.policy_violations.includes('personal_information')) {
    reasons.push('Content contains personal information that should not be shared')
  }

  if (analysis.policy_violations.includes('hate_speech')) {
    reasons.push('Content violates hate speech policy')
  }

  if (analysis.spam_score > 0.5) {
    reasons.push('Content appears to be spam or promotional')
  }

  return reasons.length > 0 ? reasons.join('. ') : 'Content violates community guidelines'
}

async function escalateToHuman(content: string, analysis: any) {
  // In practice, send to human moderation queue
  console.log('Escalating to human moderator:', {
    content: content.substring(0, 200) + '...',
    analysis,
    timestamp: new Date().toISOString()
  })
}`

import express from 'express'
import { moderateContent } from './content-moderation-agent'

const app = express()
app.use(express.json())

// Moderate forum posts
app.post('/api/posts', async (req, res) => {
  const { content, author_id, category } = req.body

  try {
    const moderation = await moderateContent(content, `forum_post_${category}`)

    if (moderation.action === 'reject') {
      return res.status(400).json({
        error: 'Content rejected',
        reason: moderation.reason,
        code: 'MODERATION_REJECTED'
      })
    }

    if (moderation.action === 'escalated') {
      // Save post but flag for human review
      const post = await savePost({
        ...req.body,
        status: 'pending_review',
        moderation_result: moderation
      })

      return res.status(202).json({
        message: 'Post submitted for review',
        post_id: post.id
      })
    }

    // Approve and save post
    const post = await savePost({
      ...req.body,
      status: 'approved',
      moderation_result: moderation
    })

    res.status(201).json({ post_id: post.id })

  } catch (error) {
    console.error('Moderation error:', error)
    res.status(500).json({ error: 'Moderation service unavailable' })
  }
})

// Moderate chat messages
app.post('/api/chat/messages', async (req, res) => {
  const { content, room_id, user_id } = req.body

  const moderation = await moderateContent(content, `chat_${room_id}`)

  if (moderation.action === 'reject') {
    return res.status(400).json({
      error: 'Message blocked',
      reason: moderation.reason
    })
  }

  // Send message to chat room
  await sendToChatRoom(room_id, {
    ...req.body,
    moderation_status: moderation.action
  })

  res.status(200).json({ status: 'sent' })
})

app.listen(3000, () => {
  console.log('Content moderation service running on port 3000')
})