403Webshell
Server IP : 3.147.158.171  /  Your IP : 216.73.216.88
Web Server : Apache/2.4.67 (Amazon Linux) OpenSSL/3.5.5
System : Linux ip-172-31-2-178.us-east-2.compute.internal 6.1.172-216.329.amzn2023.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 20 06:31:34 UTC 2026 x86_64
User : ec2-user ( 1000)
PHP Version : 8.4.21
Disable Function : NONE
MySQL : OFF  |  cURL : ON  |  WGET : ON  |  Perl : ON  |  Python : OFF  |  Sudo : ON  |  Pkexec : OFF
Directory :  /tsai/www/html/api.turmansolutions.ai/app/services/

Upload File :
current_dir [ Writeable ] document_root [ Writeable ]

 

Command :


[ Back ]     

Current File : /tsai/www/html/api.turmansolutions.ai/app/services/chat.py
import logging
import os
import re
import time
from datetime import datetime
from typing import Any, List, Dict, Optional, Tuple

from langchain_core.messages import HumanMessage, SystemMessage
from langchain_openai import ChatOpenAI
from langchain_tavily import TavilySearch

from app.appTypes import ArticleResponse, CorrectionResponse, LLModel, LLMProvider, RevisionResponse
from app.api.v1.endpoints.responses_langchain import LangChainChatProvider
from app.clients import markdown_parser as md, openai_client
from app.utilities.math_tex import convert_math
from app.services import usage_history

from dotenv import load_dotenv
load_dotenv()

# Cached TavilySearch instances keyed by (max_results, topic)
_tavily_search_cache: dict = {}


def _get_tavily_search(max_results: int = 5, topic: str = "general", search_depth: str = "basic") -> Optional[TavilySearch]:
    """Get or create cached TavilySearch instance for given parameters"""
    cache_key = f"{max_results}:{topic}:{search_depth}"
    if cache_key not in _tavily_search_cache:
        tavily_api_key = os.getenv('TAVILY_API_KEY')
        if not tavily_api_key:
            return None
        _tavily_search_cache[cache_key] = TavilySearch(
            api_key=tavily_api_key,
            max_results=max_results,
            topic=topic,
            include_answer=True,
            search_depth=search_depth,
            include_raw_content=True,
            include_images=False
        )
    return _tavily_search_cache[cache_key]


def extract_json_from_response(response_text: str) -> str:
    """
    Extract and clean JSON from Claude response that may contain:
    - Markdown code fences (```json ... ```)
    - Extra explanatory text
    - Invalid control characters
    """
    text = response_text.strip()

    # Strip markdown code fences if present
    if text.startswith('```'):
        first_newline = text.find('\n')
        if first_newline != -1:
            text = text[first_newline + 1:]
    if text.rstrip().endswith('```'):
        text = text.rstrip()[:-3]
    text = text.strip()

    # If text doesn't start with {, try to find a JSON object
    if not text.startswith('{'):
        json_obj_match = re.search(r'\{.*\}', text, re.DOTALL)
        if json_obj_match:
            text = json_obj_match.group(0)

    # Escape literal newlines/carriage returns inside JSON string values
    # (Claude often returns raw newlines in the body field, which is invalid JSON)
    result = []
    in_string = False
    for i, ch in enumerate(text):
        if ch == '"' and (i == 0 or text[i - 1] != '\\'):
            in_string = not in_string
            result.append(ch)
        elif in_string and ch == '\n':
            result.append('\\n')
        elif in_string and ch == '\r':
            result.append('\\r')
        else:
            result.append(ch)
    text = ''.join(result)

    # Clean invalid control characters (except \n, \r, \t)
    cleaned = re.sub(r'[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]', ' ', text)

    return cleaned


class ChatService:

    def search_current_events(self, query: str, max_results: int = 5, include_domains: Optional[List[str]] = None, exclude_domains: Optional[List[str]] = None, topic: str = "general", time_range: Optional[str] = None, search_depth: str = "basic", chunks_per_source: Optional[int] = None) -> List[Dict]:
        """Search for current events using Tavily Search

        Args:
            query: Search query string
            max_results: Maximum number of results to return
            include_domains: Optional list of domains to include (only these domains will be searched)
            exclude_domains: Optional list of domains to exclude from results
            topic: Search topic type - 'news' for news-focused or 'general' for general web search
            time_range: Time range filter - 'day', 'week', 'month', 'year', or None for all time
            search_depth: Search depth - 'basic' (1 credit) or 'advanced' (2 credits)
            chunks_per_source: Number of content chunks per source (1-3, only with advanced depth)

        Returns a list of search results with title, url, content, and score
        """
        try:
            # Determine if we need custom filtering (can't use cached instance)
            has_custom_filters = (
                (include_domains and len(include_domains) > 0) or
                (exclude_domains and len(exclude_domains) > 0) or
                (time_range and time_range in ['day', 'week', 'month', 'year'])
            )

            if has_custom_filters:
                # Create a fresh instance with custom filters
                tavily_api_key = os.getenv('TAVILY_API_KEY')
                if not tavily_api_key:
                    logging.warning("TAVILY_API_KEY not configured, skipping search")
                    return []

                search_params = {
                    "api_key": tavily_api_key,
                    "max_results": max_results,
                    "topic": topic,
                    "include_answer": True,
                    "search_depth": search_depth,
                    "include_raw_content": True,
                    "include_images": False
                }

                if include_domains and len(include_domains) > 0:
                    search_params["include_domains"] = include_domains
                    logging.info(f"Tavily search will ONLY include domains: {include_domains}")

                if exclude_domains and len(exclude_domains) > 0:
                    search_params["exclude_domains"] = exclude_domains
                    logging.info(f"Tavily search will exclude domains: {exclude_domains}")

                if time_range and time_range in ['day', 'week', 'month', 'year']:
                    search_params["time_range"] = time_range
                    logging.info(f"Tavily search will filter to past {time_range}")

                search_tool = TavilySearch(**search_params)
            else:
                # Use cached instance for standard searches
                search_tool = _get_tavily_search(max_results, topic, search_depth)
                if not search_tool:
                    logging.warning("TAVILY_API_KEY not configured, skipping search")
                    return []

            # Perform the search with retries. Tavily intermittently returns a
            # malformed payload (e.g. a plain string error message) or an empty
            # result set on a transient hiccup; treat both as retryable rather
            # than silently yielding zero sources (which drops the
            # _tsai_search_results meta and leaves "researched" articles
            # unsourced). See investigation in
            # docs/scike-guitar-fact-check-candidates.md.
            max_attempts = 3
            retry_delay = 1.0  # seconds, linear backoff
            last_issue = "no results"

            for attempt in range(1, max_attempts + 1):
                try:
                    if chunks_per_source is not None and search_depth == "advanced":
                        response = search_tool.invoke({"query": query, "chunks_per_source": chunks_per_source})
                    else:
                        response = search_tool.invoke(query)

                    # Extract results from response
                    if isinstance(response, dict) and 'results' in response:
                        results = response['results']
                    elif isinstance(response, list):
                        results = response
                    else:
                        last_issue = f"unexpected response format: {type(response).__name__}"
                        logging.warning(
                            f"Tavily search for '{query}' attempt {attempt}/{max_attempts}: {last_issue}"
                        )
                        results = None

                    if results:
                        logging.info(f"Found {len(results)} results for '{query}' (attempt {attempt})")
                        usage_history.record_tavily_search(search_depth=search_depth)
                        return results

                    if results is not None:
                        # Well-formed but empty — retry in case it was a transient miss
                        last_issue = "no results"
                        logging.warning(
                            f"Tavily search for '{query}' attempt {attempt}/{max_attempts}: 0 results"
                        )

                except Exception as e:
                    last_issue = f"exception: {e}"
                    logging.warning(
                        f"Tavily search for '{query}' attempt {attempt}/{max_attempts} failed: {e}"
                    )

                if attempt < max_attempts:
                    time.sleep(retry_delay * attempt)

            logging.error(
                f"Tavily search for '{query}' returned no usable results after "
                f"{max_attempts} attempts (last issue: {last_issue})"
            )
            return []

        except Exception as e:
            logging.error(f"Tavily search failed: {str(e)}")
            import traceback
            logging.error(traceback.format_exc())
            return []

    def createArticle(self, system_msg: str, human_msg: str, model: Optional[LLModel] = None, search_results: Optional[List[Dict[str, Any]]] = None, images: Optional[List[str]] = None, seed_image_instructions: Optional[str] = None) -> Tuple[ArticleResponse, LLModel, str]:
        """Create article using LangChain or fallback to OpenAI structured output.
        Returns tuple of (article_response, actual_model_used, enhanced_human_msg)

        Args:
            images: Optional list of image URLs from series for visual continuity
            seed_image_instructions: Optional user-supplied guidance for a seed
                reference image (e.g. "use this person as the main character").
                When set, the framing line appended to human_msg describes a
                user-provided reference image instead of "previous articles in
                this series" — used for single articles or Part 1 of a series.
        """

        # If search results provided, enhance the human message with context
        if search_results and len(search_results) > 0:
            search_context = "\n\n## Current Events Research:\n\n"
            for i, result in enumerate(search_results, 1):
                title = result.get('title', 'No title')
                url = result.get('url', '')
                content = result.get('content', '')
                search_context += f"**Source {i}: {title}**\n"
                search_context += f"URL: {url}\n"
                search_context += f"Content: {content}\n\n"

            human_msg = f"{human_msg}\n\n{search_context}\n\nPlease write an article incorporating this current information.\n\nIMPORTANT: You must include citations using numbered references. Use this exact format:\n- In the article body, cite each source with a clickable superscript number written as raw HTML: <sup><a href=\"URL\">[1]</a></sup>, <sup><a href=\"URL\">[2]</a></sup>, etc., placed right after the relevant statement. Use this HTML form whether the body is markdown or HTML.\n- NEVER print a raw source URL in the body text — the URL belongs only in the href and in the References list below.\n- At the end of the article, include a 'References' section: a numbered list of all sources you cited. Each entry must show the source title as a clickable link, followed by the full URL shown as visible text. For example: '1. [Source Title](URL) — URL'.\n- You don't need to use all provided sources, but you MUST keep the original source numbers. If you use Sources 1, 3, and 5, cite them as [1], [3], [5] in both the body and the References list — do not renumber them."

        # Add image framing if images are provided. A seed reference image (with
        # user instructions) gets seed-framed text; otherwise we treat the images
        # as prior-part references and use the series-continuity framing.
        if images and len(images) > 0:
            if seed_image_instructions and seed_image_instructions.strip():
                human_msg += (
                    "\n\nA reference image has been attached by the user. "
                    f"User instructions for this image: {seed_image_instructions.strip()}\n"
                    "Use it to ground your character descriptions, setting, and visual "
                    "details in the article — and let it inform the featured/secondary "
                    "image descriptions you write."
                )
            else:
                human_msg += "\n\nThe following images are from previous articles in this series. Use them as inspiration for visual continuity — maintain similar artistic style, character appearances, color palette, and setting details in your image descriptions."

        # Append human message to log file
        model_info = f"{model.provider.value}: {model.name}" if model else "Default model"
        search_enabled = "Yes" if search_results and len(search_results) > 0 else "No"

        log_append = f"""
---

## Human Message

{human_msg}

## Additional Context
- Model: {model_info}
- Current events research: {search_enabled}
"""

        try:
            with open('logs/article-instructions.md', 'a') as file:
                file.write(log_append)
        except Exception as e:
            logging.warning(f"Failed to append to article instructions log: {e}")

        # Build user message content (text or vision format with images)
        def _build_user_content(msg: str, imgs: Optional[List[str]] = None):
            if imgs and len(imgs) > 0:
                content: List[Dict[str, Any]] = [{"type": "text", "text": msg}]
                for img_url in imgs:
                    if img_url and '.test' not in img_url:
                        content.append({"type": "image_url", "image_url": {"url": img_url}})
                return content
            return msg

        user_content = _build_user_content(human_msg, images)

        # If model is provided, use LangChain with structured output via OpenAI
        if model and model.provider == LLMProvider.OPEN_AI:
            # Use OpenAI structured output for best results
            # Build params dict to avoid passing None values
            params = {
                "model": model.name,
                "messages": [
                    {"role": "system", "content": system_msg},
                    {"role": "user", "content": user_content},
                ],
                "response_format": ArticleResponse,
            }

            # o1 and gpt-5 models have restrictions on temperature and other params
            if 'o1' in model.name or 'gpt-5' in model.name:
                # These models only support temperature=1 (default)
                # o1 models also don't support max_tokens
                params["temperature"] = 1
                # gpt-5 uses max_completion_tokens (not max_tokens); o1 omits it
                if 'gpt-5' in model.name and model.max_tokens is not None:
                    params["max_completion_tokens"] = model.max_tokens
            else:
                # Standard models support custom temperature and max_tokens
                params["temperature"] = model.temperature or 0.7
                if model.max_tokens is not None:
                    params["max_tokens"] = model.max_tokens

            completion = openai_client.beta.chat.completions.parse(**params)
            usage_history.record_openai_completion(completion, model=model.name, operation='article')
            resp = completion.choices[0].message.parsed
            if resp is None:
                raise ValueError("Failed to parse OpenAI response")
            return (resp, model, human_msg)
        elif model and model.provider == LLMProvider.CLAUDE:
            # Ensure sufficient max_tokens for article generation
            # Series articles need more tokens due to HTML-formatted body in structured output
            min_tokens = 64000
            if not model.max_tokens or model.max_tokens < min_tokens:
                model = model.model_copy(update={"max_tokens": min_tokens})
            logging.info(f"Claude article generation: model={model.name}, max_tokens={model.max_tokens}")
            chat_model = LangChainChatProvider.create_chat_model(model)
            # include_raw=True so we can read usage_metadata off the underlying
            # AIMessage — with_structured_output otherwise discards it.
            structured_model = chat_model.with_structured_output(ArticleResponse, include_raw=True)
            sys_msg = SystemMessage(content=system_msg)
            if images and len(images) > 0:
                vision_content: List[Dict[str, Any]] = [{"type": "text", "text": human_msg}]
                for img_url in images:
                    if img_url and '.test' not in img_url:
                        vision_content.append({"type": "image_url", "image_url": {"url": img_url}})
                human = HumanMessage(content=vision_content)  # type: ignore
            else:
                human = HumanMessage(content=human_msg)
            try:
                raw_result = structured_model.invoke([sys_msg, human])
            except Exception as e:
                logging.error(f"Claude structured-output call failed: {e}")
                raise
            usage_history.record_langchain_message(
                raw_result.get('raw'), provider=LLMProvider.CLAUDE.value,
                model=model.name, operation='article',
            )
            response = raw_result['parsed']
            return (response, model, human_msg)  # type: ignore[return-value]
        else:
            # Default fallback to hardcoded gpt-5.5
            default_model = LLModel(
                provider=LLMProvider.OPEN_AI,
                name="gpt-5.5",
                temperature=0.7,
                max_tokens=None
            )
            completion = openai_client.beta.chat.completions.parse(
                model="gpt-5.5",
                messages=[
                    {"role": "system", "content": system_msg},
                    {"role": "user", "content": user_content},
                ],
                response_format=ArticleResponse,
            )
            usage_history.record_openai_completion(completion, model="gpt-5.5", operation='article')
            resp = completion.choices[0].message.parsed
            if resp is None:
                raise ValueError("Failed to parse OpenAI response")
            return (resp, default_model, human_msg)

    def chat(self, instructions: str, content: str) -> str:
        model = ChatOpenAI(model="gpt-5.5")
        sys_msg = SystemMessage(content=instructions)
        human_msg = HumanMessage(content=content)
        my_response = model.invoke([sys_msg, human_msg])
        usage_history.record_langchain_message(
            my_response, provider=LLMProvider.OPEN_AI.value, model="gpt-5.5", operation='chat',
        )
        return my_response.content if isinstance(my_response.content, str) else str(my_response.content)

    def fetch_comment(self, instructions: str, content: str, images: List[str], model: Optional[LLModel] = None, operation: str = 'comment') -> str:
        """Generate comment using LangChain or fallback to default OpenAI"""

        # Use provided model or default to gpt-5.5
        if model:
            chat_model = LangChainChatProvider.create_chat_model(model)
            usage_provider = model.provider.value
            usage_model = model.name
        else:
            chat_model = ChatOpenAI(model="gpt-5.5")
            usage_provider = LLMProvider.OPEN_AI.value
            usage_model = "gpt-5.5"

        sys_msg = SystemMessage(content=instructions)

        # Handle vision content if images are present
        vision_content: List[Dict[str, Any]] = [{"type": "text", "text": content}]
        if images:
            for image in images:
                # Skip None images
                if image is None:
                    logging.warning('None image skipped.')
                    continue

                myIndex = image.find('.test')
                if myIndex > -1:
                    logging.warning('localhost image skipped.')
                    continue

                myObj: Dict[str, Any] = {"type": "image_url", "image_url": {"url": image}}
                vision_content.append(myObj)

        human_msg = HumanMessage(
            content=vision_content  # type: ignore
        )

        my_response = chat_model.invoke([sys_msg, human_msg])
        usage_history.record_langchain_message(
            my_response, provider=usage_provider, model=usage_model, operation=operation,
        )
        content_str = my_response.content if isinstance(my_response.content, str) else str(my_response.content)
        logging.info(f'AI comment: {content_str[:600]}')
        html = convert_math(md.render(content_str))
        # logging.info(f'MARKDOWN: {html}')
        # logging.info(f'MARKDOWN: {html[:200]}')
        return html

    def _calculate_date_recency(self, published_date_str: str, today: datetime) -> str:
        """Calculate how recent a date is relative to today

        Args:
            published_date_str: Date string from Tavily (various formats possible)
            today: Current datetime for comparison

        Returns:
            Human-readable recency string (e.g., "3 days ago - CURRENT EVENT")
        """
        if not published_date_str or published_date_str == 'Unknown date':
            return ''

        try:
            # Try parsing ISO format (YYYY-MM-DD)
            if len(published_date_str) >= 10:
                published_date = datetime.fromisoformat(published_date_str[:10])
            else:
                return ''

            delta = today - published_date
            days = delta.days

            # Calculate recency label
            if days < 0:
                return f" (future date: {abs(days)} days from now)"
            elif days == 0:
                return " (TODAY - CURRENT EVENT)"
            elif days == 1:
                return " (yesterday - CURRENT EVENT)"
            elif days < 7:
                return f" ({days} days ago - CURRENT EVENT)"
            elif days < 14:
                return f" ({days} days ago - RECENT)"
            elif days < 30:
                weeks = days // 7
                return f" ({weeks} week{'s' if weeks > 1 else ''} ago - RECENT)"
            elif days < 90:
                months = days // 30
                return f" ({months} month{'s' if months > 1 else ''} ago)"
            else:
                months = days // 30
                return f" ({months} months ago)"

        except (ValueError, TypeError) as e:
            logging.debug(f"Could not parse date '{published_date_str}': {e}")
            return ''

    def fact_check_article(self, article_title: str, article_body: str, images: List[str], original_model: Optional[LLModel] = None, search_results: Optional[List[Dict[str, Any]]] = None) -> Tuple[str, str, str]:
        """Fact-check an article using the opposite AI provider

        Args:
            article_title: Title of the article to fact-check
            article_body: Body content of the article
            images: List of image URLs associated with the article
            original_model: The AI model used to create the article
            search_results: Optional Tavily search results that informed the article

        Returns:
            tuple: (fact_check_html, model_name, provider_name)
        """

        # Determine fact-checking model (opposite provider)
        if original_model and original_model.provider == LLMProvider.OPEN_AI:
            # Use Claude for fact-checking
            fact_check_model = LLModel(
                provider=LLMProvider.CLAUDE,
                name="claude-sonnet-4-6",
                temperature=0.5
            )
        else:
            # Use OpenAI for fact-checking
            fact_check_model = LLModel(
                provider=LLMProvider.OPEN_AI,
                name="gpt-5.5",
                temperature=0.5
            )

        # Get today's date for temporal context
        today = datetime.now()
        today_str = today.strftime('%Y-%m-%d')
        today_readable = today.strftime('%B %d, %Y')

        has_sources = bool(search_results)

        # Create fact-checking instructions
        if has_sources:
            instructions = f"""You are reviewing an article to assess its accuracy against source material.

TODAY'S DATE: {today_readable} ({today_str})

TRUST THE SOURCES:
- The sources below come from Tavily Search, a real-time web search API that retrieves current news
- When reputable news outlets (ABC, BBC, Reuters, AP, Investopedia, etc.) report events, presume those events occurred
- Breaking news may seem extraordinary or unprecedented - this does NOT make it false
- Do NOT apply your own judgment about what "should" be possible or probable
- Your role is to verify the article matches the sources, NOT to second-guess the sources themselves
- Your training data has a cutoff, but these sources are MORE RECENT - trust them over your priors

ASSESSMENT CRITERIA:
1. Does the article accurately represent information from the provided sources?
2. Are there claims that directly CONTRADICT the source material?
3. Are there significant factual details with NO source support at all?

RESPONSE GUIDELINES:
- If the article accurately represents its sources, acknowledge this briefly (1-2 sentences is fine)
- Only flag genuine factual discrepancies between the article text and the source content
- Do NOT criticize editorial style, dramatic phrasing, headline tone, or minor citation formatting
- For developing/breaking news, it's normal for details to evolve - this is not an error
- Be proportionate: minor issues deserve brief mention at most, not lengthy criticism

Keep your response concise (2-3 short paragraphs maximum)."""
        else:
            instructions = f"""You are reviewing an article for factual accuracy using your own knowledge.

TODAY'S DATE: {today_readable} ({today_str})

CONTEXT:
- This article was written from the author's general knowledge, not from real-time research
- No external sources were gathered for it, and none are expected
- Do NOT remark on the absence of citations or sources - that is by design for this piece
- Your job is to spot factual errors, not to ask for sourcing

ASSESSMENT CRITERIA:
1. Are there clear factual errors a knowledgeable reader would catch (wrong dates, misattributed quotes, incorrect names, mischaracterized concepts)?
2. Are there internal contradictions within the article itself?
3. Are there claims that are well-established as false, or that confidently overstate contested matters?

RESPONSE GUIDELINES:
- If the article reads as accurate to your knowledge, say so briefly (1-2 sentences is fine)
- Only flag genuine factual problems - not stylistic choices, opinions, interpretations, or dramatic phrasing
- For evergreen / instructional / opinion / creative content, accuracy is about the factual scaffolding, not the take
- Be proportionate: minor issues deserve brief mention at most, not lengthy criticism

Keep your response concise (2-3 short paragraphs maximum)."""

        content = f"**Article Title:** {article_title}\n\n**Article Content:**\n{article_body}"

        # If search results were provided, include them in the fact-check context
        content += self._build_source_material_block(search_results, today)

        # Generate fact-check using opposite provider
        fact_check_content = self.fetch_comment(
            instructions, content, images, model=fact_check_model, operation='fact_check'
        )

        model_name = fact_check_model.name
        provider_name = fact_check_model.provider.value

        # Add magnifying glass emoji at the start with larger size
        fact_check_with_icon = f'<span style="font-size: 1.5em;">🔍</span> {fact_check_content}'

        logging.info(f'Fact-check generated using {provider_name} {model_name}')
        return fact_check_with_icon, model_name, provider_name

    def _build_source_material_block(self, search_results: Optional[List[Dict[str, Any]]], today: datetime) -> str:
        """Build the '## Source Material Used' context block from stored Tavily results.

        Returns an empty string when there are no results. Shared by the fact-check
        and corrections paths so both reason over the exact same sources.
        """
        if not search_results or len(search_results) == 0:
            return ""

        block = "\n\n## Source Material Used\n\nThe following sources were used to inform this article:\n\n"
        for i, result in enumerate(search_results, 1):
            title = result.get('title', 'No title')
            url = result.get('url', '')

            # Use raw_content if available, fall back to snippet
            raw_content = result.get('raw_content', '')
            snippet = result.get('content', '')

            # Metadata
            published_date = result.get('published_date', 'Unknown date')
            score = result.get('score', 0.0)
            score_percentage = int(score * 100)

            # Calculate date recency for temporal context
            recency_label = self._calculate_date_recency(published_date, today)

            # Images (may not be available through LangChain wrapper)
            source_images = result.get('images', [])

            block += f"**Source {i}: {title}**\n"
            block += f"URL: {url}\n"
            block += f"Published: {published_date}{recency_label}\n"
            block += f"Relevance Score: {score_percentage}%\n"

            # Include snippet for quick reference
            if snippet:
                block += f"Summary: {snippet}\n\n"

            # Include full content for detailed verification
            if raw_content:
                # Truncate if extremely long (over 10,000 chars)
                truncated_content = raw_content[:10000] + "..." if len(raw_content) > 10000 else raw_content
                block += f"Full Content:\n{truncated_content}\n\n"

            # Include images if available
            if source_images and len(source_images) > 0:
                block += f"Related Images ({len(source_images)}):\n"
                for img_url in source_images[:3]:  # Limit to first 3 images per source
                    block += f"  - {img_url}\n"
                block += "\n"

        block += "\nVERIFICATION NOTES:\n"
        block += "- The sources above ARE the verification - if they report it, it happened\n"
        block += "- Sources marked 'CURRENT EVENT' or 'RECENT' are real-time news from the past few weeks\n"
        block += "- Higher relevance scores indicate more relevant/reliable sources\n"
        block += "- Your task: confirm the article accurately represents these sources, not evaluate whether the events 'should' have happened\n"
        return block

    def generate_corrections(self, article_title: str, article_body: str, fact_check_text: str, images: List[str], original_model: Optional[LLModel] = None, search_results: Optional[List[Dict[str, Any]]] = None, model_override: Optional[LLModel] = None) -> Tuple[str, Optional[str], str, str]:
        """Produce a corrections follow-up to an editorial fact-check.

        Reads the article, the fact-check comment, and (when available) the same
        Tavily sources the article was written from, then decides whether the body
        needs factual edits. Always returns a comment documenting the outcome.

        Args:
            article_title: Title of the article
            article_body: Current HTML body of the article
            fact_check_text: Plain-text content of the fact-check comment to act on
            images: Image URLs associated with the article (vision context)
            original_model: The AI model used to create the article (drives provider choice)
            search_results: Stored Tavily results that informed the article
            model_override: When set, forces this model for the corrections regardless of
                the opposite-provider rule (used to let callers pick a specific LLM)

        Returns:
            tuple: (comment_html, revised_body_or_None, model_name, provider_name)
        """

        # An explicit override wins; otherwise use the opposite provider, matching
        # the fact-check's selection logic.
        if model_override:
            corrections_model = LLModel(
                provider=model_override.provider,
                name=model_override.name,
                temperature=0.5
            )
        elif original_model and original_model.provider == LLMProvider.OPEN_AI:
            corrections_model = LLModel(
                provider=LLMProvider.CLAUDE,
                name="claude-sonnet-4-6",
                temperature=0.5
            )
        else:
            corrections_model = LLModel(
                provider=LLMProvider.OPEN_AI,
                name="gpt-5.5",
                temperature=0.5
            )

        today = datetime.now()
        today_readable = today.strftime('%B %d, %Y')
        today_str = today.strftime('%Y-%m-%d')

        instructions = f"""You are the editor applying corrections to a published article in response to an editorial fact-check.

TODAY'S DATE: {today_readable} ({today_str})

TRUST THE SOURCES:
- When source material is provided below, it comes from real-time web search and is MORE RECENT than your training cutoff - trust it over your own priors.
- Breaking or extraordinary news is not "wrong" simply because it is surprising. Do not second-guess the sources themselves.

YOUR TASK:
1. Read the article, the fact-check, and any source material.
2. Decide whether the body needs FACTUAL corrections. Only correct genuine factual errors the fact-check identified that the sources (or well-established knowledge) support fixing - wrong dates, names, figures, misattributed quotes, claims that contradict the sources.
3. NEVER edit for style, tone, dramatic phrasing, headline voice, opinion, or citation formatting. Those are not errors.

OUTPUT (structured):
- edits_needed: true only if you are making at least one factual correction to the body.
- revised_body: when edits_needed is true, the COMPLETE corrected article body as HTML, preserving the original structure, formatting, and everything that was already correct - change only what must change. Leave null when edits_needed is false.
- comment: a short reader-facing note, broken into 2-4 short paragraphs.
  - If you made edits, summarize exactly what was changed and why, tying each correction to the fact-check finding. Give each correction (or closely related group of corrections) its own paragraph instead of stringing them all into one block.
  - If no edits were warranted, briefly state that the article stands as written and why the fact-check raised no actionable factual errors.

WRITING STYLE (for the comment):
- Break the note into multiple short paragraphs, each separated by a BLANK LINE. Never return one long undivided paragraph.
- Plain prose only - no headings."""

        content = f"**Article Title:** {article_title}\n\n**Article Content (HTML):**\n{article_body}"
        content += f"\n\n## Editorial Fact-Check To Act On\n\n{fact_check_text}\n"
        content += self._build_source_material_block(search_results, today)

        resp = self._invoke_structured(corrections_model, instructions, content, images, CorrectionResponse)

        model_name = corrections_model.name
        provider_name = corrections_model.provider.value

        comment_html = convert_math(md.render(resp.comment))
        comment_with_icon = f'<span style="font-size: 1.5em;">📝</span> {comment_html}'

        revised_body = resp.revised_body if resp.edits_needed else None

        logging.info(
            f'Corrections generated using {provider_name} {model_name} '
            f'(edits_needed={resp.edits_needed})'
        )
        return comment_with_icon, revised_body, model_name, provider_name

    def revise_article_body(self, article_title: str, article_body: str, revision_instructions: str,
                            images: Optional[List[str]] = None,
                            model: Optional[LLModel] = None) -> Tuple[str, str]:
        """Rewrite an article body according to an editor's instructions.

        Unlike generate_corrections, this makes the edit the editor asked for —
        style, structure, and tone are all fair game — and it always revises.
        Use generate_corrections when the driver is a fact-check.

        Args:
            article_title: Title of the article
            article_body: Current HTML body of the article
            revision_instructions: What the editor wants changed, in their words
            images: Image URLs associated with the article (vision context)
            model: Model to use; defaults to Claude Sonnet 4.6

        Returns:
            tuple: (revised_body_html, summary_of_changes)
        """
        revision_model = model or LLModel(
            provider=LLMProvider.CLAUDE, name="claude-sonnet-4-6", temperature=0.5
        )

        instructions = """You are an editor revising a published article on behalf of its author.

YOUR TASK:
1. Read the article and the requested revision.
2. Apply exactly the revision requested — nothing more. Preserve the author's voice, the article's structure, and every part the request does not touch.
3. Keep all existing inline citations intact and in their canonical form. Never invent new facts, figures, quotes, or citations; if the request needs information the article does not contain, work with what is there and note the gap in your summary.

OUTPUT (structured):
- revised_body: the COMPLETE revised article body as HTML. Never return a fragment, a diff, or a description of the change — return the whole body, ready to publish.
- summary: one or two sentences stating what you changed, for the editor to read in chat.

FORMATTING:
- Do not repeat the article title at the top of the body; the site renders it separately.
- Do not add <img> or <figure> tags. Images are managed by the publishing pipeline."""

        content = (f"**Article Title:** {article_title}\n\n"
                   f"**Article Content (HTML):**\n{article_body}\n\n"
                   f"## Requested Revision\n\n{revision_instructions}\n")

        resp = self._invoke_structured(
            revision_model, instructions, content, images or [], RevisionResponse,
            operation='revision',
        )
        logging.info(
            f'Article revision generated using {revision_model.provider.value} '
            f'{revision_model.name}: {resp.summary[:200]}'
        )
        return resp.revised_body, resp.summary

    def _invoke_structured(self, model: LLModel, instructions: str, content: str, images: List[str], schema, operation: str = 'corrections'):
        """Invoke a model with structured output, mirroring createArticle's provider handling.

        Returns an instance of `schema`. Supports vision content via image URLs.
        """
        def _vision_human(text: str) -> HumanMessage:
            vision_content: List[Dict[str, Any]] = [{"type": "text", "text": text}]
            for image in images or []:
                if image and '.test' not in image:
                    vision_content.append({"type": "image_url", "image_url": {"url": image}})
            return HumanMessage(content=vision_content)  # type: ignore

        if model.provider == LLMProvider.CLAUDE:
            min_tokens = 64000
            if not model.max_tokens or model.max_tokens < min_tokens:
                model = model.model_copy(update={"max_tokens": min_tokens})
            chat_model = LangChainChatProvider.create_chat_model(model)
            # include_raw=True to recover usage_metadata (see createArticle).
            structured_model = chat_model.with_structured_output(schema, include_raw=True)
            sys_msg = SystemMessage(content=instructions)
            raw_result = structured_model.invoke([sys_msg, _vision_human(content)])
            usage_history.record_langchain_message(
                raw_result.get('raw'), provider=LLMProvider.CLAUDE.value,
                model=model.name, operation=operation,
            )
            return raw_result['parsed']

        # OpenAI structured-output path
        user_content: Any = content
        if images:
            user_content = [{"type": "text", "text": content}]
            for image in images:
                if image and '.test' not in image:
                    user_content.append({"type": "image_url", "image_url": {"url": image}})
        completion = openai_client.beta.chat.completions.parse(
            model=model.name,
            messages=[
                {"role": "system", "content": instructions},
                {"role": "user", "content": user_content},
            ],
            response_format=schema,
        )
        usage_history.record_openai_completion(completion, model=model.name, operation=operation)
        parsed = completion.choices[0].message.parsed
        if parsed is None:
            raise ValueError("Failed to parse structured response")
        return parsed

Youez - 2016 - github.com/yon3zu
LinuXploit