| Server IP : 3.147.158.171 / Your IP : 216.73.216.88 Web Server : Apache/2.4.67 (Amazon Linux) OpenSSL/3.5.5 System : Linux ip-172-31-2-178.us-east-2.compute.internal 6.1.172-216.329.amzn2023.x86_64 #1 SMP PREEMPT_DYNAMIC Wed May 20 06:31:34 UTC 2026 x86_64 User : ec2-user ( 1000) PHP Version : 8.4.21 Disable Function : NONE MySQL : OFF | cURL : ON | WGET : ON | Perl : ON | Python : OFF | Sudo : ON | Pkexec : OFF Directory : /tsai/www/html/api.turmansolutions.ai/app/services/ |
Upload File : |
import os
from typing import List, Optional
import logging
from fastapi import HTTPException
import requests
import httpx
from ..appTypes import Character, LLModel, ContentSchedule
from .schedule_service import ScheduleService
from .site_config_service import SiteConfigService
import json
from dotenv import load_dotenv
load_dotenv()
DRUPAL_HOST = os.getenv('DRUPAL_HOST')
DRUPAL_API = os.getenv('DRUPAL_API')
class DrupalService:
async def login(self, session):
login_url = f'{DRUPAL_HOST}/user/login?_format=json'
login_payload = {
'name': os.getenv('DRUPAL_USER'),
'pass': os.getenv('DRUPAL_PASSWORD')
}
login_response = session.post(login_url, json=login_payload)
if login_response.status_code != 200:
msg = f"Failed to log in: {login_response.status_code} {login_response.text}"
logging.info(msg)
raise HTTPException(500, 'Unable to verify credentials.')
return login_response.json()
async def logout(self, session, logout_token):
logout_url = f'{DRUPAL_HOST}/user/logout?_format=json&token={logout_token}'
logout_response = session.post(logout_url)
if logout_response.status_code not in [200, 204]:
msg = f"Failed to log out: {logout_response.status_code} {logout_response.text}"
logging.info(msg)
raise HTTPException(500, 'Operation failed.')
return logout_response
async def get_csrf_token(self, session):
csrf_token_url = f"{DRUPAL_HOST}/session/token"
csrf_response = session.get(csrf_token_url)
if csrf_response.status_code != 200:
msg = f"Failed to get CSRF token: {csrf_response.status_code} {csrf_response.text}"
logging.info(msg)
raise HTTPException(500, 'Unable to verify credentials.')
return csrf_response.text
def fetch_characters(self, limit=10):
endpoint = f"{DRUPAL_HOST}{DRUPAL_API}/node/character?include=uid&page[limit]={limit}"
try:
response = requests.get(endpoint)
if response.status_code == 200:
data = response.json()
articles = data.get('data', [])
return articles
else:
logging.info(
f"Error: Received status code {response.status_code}")
return []
except requests.exceptions.RequestException as e:
logging.info(f"Error: {e}")
return []
def fetch_all_characters(self):
"""Fetch ALL characters using auto-pagination to handle Drupal limits"""
all_characters = []
page_offset = 0
page_limit = 50 # Use reasonable page size
try:
while True:
endpoint = f"{DRUPAL_HOST}{DRUPAL_API}/node/character?include=uid&page[limit]={page_limit}&page[offset]={page_offset}"
response = requests.get(endpoint)
if response.status_code != 200:
logging.info(f"Error fetching characters page: {response.status_code} {response.text}")
break
data = response.json()
page_characters = data.get('data', [])
# If no characters returned, we've reached the end
if not page_characters:
break
all_characters.extend(page_characters)
# Check if we got fewer characters than requested (last page)
if len(page_characters) < page_limit:
break
page_offset += page_limit
logging.info(f"Fetched {len(all_characters)} total characters across {(page_offset // page_limit) + 1} pages")
return all_characters
except requests.exceptions.RequestException as e:
logging.info(f"Error during auto-pagination: {e}")
return all_characters # Return what we got so far
def fetch_shared_characters(self):
"""Fetch ALL shared characters using auto-pagination, filtering by field_shared=1"""
all_characters = []
page_offset = 0
page_limit = 50
try:
while True:
endpoint = f"{DRUPAL_HOST}{DRUPAL_API}/node/character?filter[field_shared]=1&include=uid&page[limit]={page_limit}&page[offset]={page_offset}"
response = requests.get(endpoint)
if response.status_code != 200:
logging.info(f"Error fetching shared characters page: {response.status_code} {response.text}")
break
data = response.json()
page_characters = data.get('data', [])
if not page_characters:
break
all_characters.extend(page_characters)
if len(page_characters) < page_limit:
break
page_offset += page_limit
logging.info(f"Fetched {len(all_characters)} shared characters across {(page_offset // page_limit) + 1} pages")
return all_characters
except requests.exceptions.RequestException as e:
logging.info(f"Error during shared characters auto-pagination: {e}")
return all_characters
def fetch_characters_by_uid(self, drupal_target_id: int):
"""Fetch all characters owned by a specific Drupal user (by internal target_id), with auto-pagination."""
all_characters = []
page_offset = 0
page_limit = 50
try:
while True:
endpoint = (
f"{DRUPAL_HOST}{DRUPAL_API}/node/character"
f"?filter[uid.meta.drupal_internal__target_id]={drupal_target_id}"
f"&include=uid"
f"&page[limit]={page_limit}&page[offset]={page_offset}"
)
response = requests.get(endpoint)
if response.status_code != 200:
logging.info(f"Error fetching characters by target_id: {response.status_code} {response.text}")
break
data = response.json()
page_characters = data.get('data', [])
if not page_characters:
break
all_characters.extend(page_characters)
if len(page_characters) < page_limit:
break
page_offset += page_limit
logging.info(f"Fetched {len(all_characters)} characters for target_id {drupal_target_id}")
return all_characters
except requests.exceptions.RequestException as e:
logging.info(f"Error fetching characters by target_id: {e}")
return all_characters
def fetch_system_wide_characters(self):
"""Fetch all characters owned by the fapi Drupal user (system-wide characters)."""
fapi_target_id = os.getenv('FAPI_DRUPAL_TARGET_ID')
if not fapi_target_id:
logging.warning("FAPI_DRUPAL_TARGET_ID not configured")
return []
return self.fetch_characters_by_uid(int(fapi_target_id))
def _fetch_character_by_slug_and_target(self, slug: str, target_id: int):
endpoint = (
f"{DRUPAL_HOST}{DRUPAL_API}/node/character"
f"?filter[field_slug]={slug}"
f"&filter[uid.meta.drupal_internal__target_id]={target_id}"
f"&include=uid"
)
try:
response = requests.get(endpoint)
if response.status_code == 200:
characters = response.json().get('data', [])
if characters:
return characters[0]
return None
logging.info(
f"Error fetching character by slug '{slug}' for target_id {target_id}: "
f"{response.status_code}"
)
return None
except requests.exceptions.RequestException as e:
logging.info(
f"Error fetching character by slug '{slug}' for target_id {target_id}: {e}"
)
return None
def fetch_character_by_slug(self, slug: str, site):
"""Fetch a character by slug, scoped to the requesting site.
Falls back to the FAPI-owned (system-wide) character with the same
slug so cross-site characters remain visible from any site.
"""
site_target_id = getattr(site, 'drupal_target_id', None) if site else None
if site_target_id is not None:
character = self._fetch_character_by_slug_and_target(slug, int(site_target_id))
if character:
return character
fapi_target_id = os.getenv('FAPI_DRUPAL_TARGET_ID')
if fapi_target_id and (site_target_id is None or int(fapi_target_id) != int(site_target_id)):
return self._fetch_character_by_slug_and_target(slug, int(fapi_target_id))
return None
def fetch_character_by_id(self, character_id: str):
"""Fetch a single character by its Drupal node ID"""
endpoint = f"{DRUPAL_HOST}{DRUPAL_API}/node/character/{character_id}?include=uid"
try:
response = requests.get(endpoint)
if response.status_code == 200:
data = response.json()
return data.get('data')
elif response.status_code == 404:
logging.info(f"Character not found: {character_id}")
return None
else:
logging.info(f"Error fetching character {character_id}: {response.status_code} {response.text}")
return None
except requests.exceptions.RequestException as e:
logging.info(f"Error fetching character {character_id}: {e}")
return None
def get_character_uid(self, character_data) -> Optional[str]:
"""Extract the owning user's UUID from a Drupal JSON:API character node."""
uid_rel = character_data.get('relationships', {}).get('uid', {}).get('data', {})
return uid_rel.get('id')
def filter_characters(self, characters: List[Character], site_id, wp_user_data, debug=False):
"""Filter characters to only include enabled ones."""
out_characters = []
for character in characters:
if not character.enabled:
logging.info(f'character not enabled: {character.name}')
continue
out_characters.append(character)
return out_characters
def selectCharacters(self, characters, field='commentsPerDay'):
"""Select characters whose calendar schedule matches the current time.
Args:
characters: List of Character objects
field: String field name ('commentsPerDay' or 'articlesPerDay'),
used to pick which schedule applies to this content type
Returns:
List of selected Character objects. A character is selected only
if it has the relevant calendar schedule and that schedule matches
now; characters with no schedule never post.
"""
out_characters = []
schedule_service = ScheduleService()
# Determine which schedule field to use based on content type
schedule_field = 'articleSchedule' if field == 'articlesPerDay' else 'commentSchedule'
for character in characters:
schedule = getattr(character, schedule_field, None)
if schedule_service.should_post_now(schedule):
logging.info(f"{character.name} selected via calendar schedule")
out_characters.append(character)
return out_characters
def _parse_schedule(self, schedule_data) -> Optional[ContentSchedule]:
"""Parse schedule data from Drupal JSON into ContentSchedule object.
Args:
schedule_data: Dict with schedule configuration or None
Returns:
ContentSchedule object or None
"""
if not schedule_data:
return None
# Both frontend and backend now use camelCase
return ContentSchedule(
daysOfWeek=schedule_data.get('daysOfWeek'),
daysOfMonth=schedule_data.get('daysOfMonth'),
preferredHours=schedule_data.get('preferredHours'),
)
def create_character(self, site, character_data):
body = character_data.get('attributes', {}).get(
'body', {}).get('value', '')
body = json.loads(body)
owner_uuid = self.get_character_uid(character_data)
site_id = ''
fapi_uuid = os.getenv('FAPI_DRUPAL_UUID')
is_system_wide = bool(fapi_uuid and owner_uuid == fapi_uuid)
if owner_uuid and not is_system_wide:
site_id = SiteConfigService().get_slug_by_drupal_uuid(owner_uuid) or ''
shared = character_data.get('attributes', {}).get('field_shared', True)
slug = character_data.get('attributes', {}).get(
'field_slug', '')
storage_id = character_data.get('id', '')
mod_date = character_data.get('attributes', {}).get('changed', '')
# print('body is', body)
# Extract model data if present
model = None
if "model" in body and body["model"] is not None:
model_data = body["model"]
model = LLModel(**model_data)
# Extract schedule data if present
article_schedule = self._parse_schedule(body.get("articleSchedule"))
comment_schedule = self._parse_schedule(body.get("commentSchedule"))
my_character_data = {
"slug": slug,
"name": body["name"],
"email": body["email"],
"description": body["description"],
"enabled": True if body["enabled"] == True else False,
"date": mod_date,
"siteId": site_id,
"storageId": storage_id,
"model": model,
"researchCurrentEvents": body.get("researchCurrentEvents", False),
"createFeaturedImage": body.get("createFeaturedImage", False),
"createSecondaryImage": body.get("createSecondaryImage", False),
"enableFactChecking": body.get("enableFactChecking", False),
"enableCorrections": body.get("enableCorrections", False),
"includePreviousArticles": body.get("includePreviousArticles", True),
"searchQuery": body.get("searchQuery", None),
"searchDomainsIncluded": body.get("searchDomainsIncluded", None),
"searchDomainsExcluded": body.get("searchDomainsExcluded", None),
"searchTopicType": body.get("searchTopicType", "general"),
"searchTimeRange": body.get("searchTimeRange", None),
"maxSearchResults": body.get("maxSearchResults", 5),
"searchDepth": body.get("searchDepth", "basic"),
"chunksPerSource": body.get("chunksPerSource", 3),
"articleSchedule": article_schedule,
"commentSchedule": comment_schedule,
"shared": shared,
"systemWide": is_system_wide,
"history": body.get("history", None),
}
character = Character(**my_character_data)
return character
def fetch_all_widgets(self):
"""Fetch ALL widgets using auto-pagination to handle Drupal limits"""
all_widgets = []
page_offset = 0
page_limit = 50
try:
while True:
endpoint = f"{DRUPAL_HOST}{DRUPAL_API}/node/widget?include=uid&page[limit]={page_limit}&page[offset]={page_offset}"
response = requests.get(endpoint)
if response.status_code != 200:
logging.info(f"Error fetching widgets page: {response.status_code} {response.text}")
break
data = response.json()
page_widgets = data.get('data', [])
if not page_widgets:
break
all_widgets.extend(page_widgets)
if len(page_widgets) < page_limit:
break
page_offset += page_limit
logging.info(f"Fetched {len(all_widgets)} total widgets across {(page_offset // page_limit) + 1} pages")
return all_widgets
except requests.exceptions.RequestException as e:
logging.info(f"Error during widget auto-pagination: {e}")
return all_widgets
def fetch_widgets_by_uid(self, drupal_target_id: int):
"""Fetch all widgets owned by a specific Drupal user (by internal target_id), with auto-pagination."""
all_widgets = []
page_offset = 0
page_limit = 50
try:
while True:
endpoint = (
f"{DRUPAL_HOST}{DRUPAL_API}/node/widget"
f"?filter[uid.meta.drupal_internal__target_id]={drupal_target_id}"
f"&include=uid"
f"&page[limit]={page_limit}&page[offset]={page_offset}"
)
response = requests.get(endpoint)
if response.status_code != 200:
logging.info(f"Error fetching widgets by target_id: {response.status_code} {response.text}")
break
data = response.json()
page_widgets = data.get('data', [])
if not page_widgets:
break
all_widgets.extend(page_widgets)
if len(page_widgets) < page_limit:
break
page_offset += page_limit
logging.info(f"Fetched {len(all_widgets)} widgets for target_id {drupal_target_id}")
return all_widgets
except requests.exceptions.RequestException as e:
logging.info(f"Error fetching widgets by target_id: {e}")
return all_widgets
def fetch_shared_widgets(self):
"""Fetch ALL shared widgets using auto-pagination, filtering by field_shared2=1"""
all_widgets = []
page_offset = 0
page_limit = 50
try:
while True:
endpoint = f"{DRUPAL_HOST}{DRUPAL_API}/node/widget?filter[field_shared2]=1&include=uid&page[limit]={page_limit}&page[offset]={page_offset}"
response = requests.get(endpoint)
if response.status_code != 200:
logging.info(f"Error fetching shared widgets page: {response.status_code} {response.text}")
break
data = response.json()
page_widgets = data.get('data', [])
if not page_widgets:
break
all_widgets.extend(page_widgets)
if len(page_widgets) < page_limit:
break
page_offset += page_limit
logging.info(f"Fetched {len(all_widgets)} shared widgets across {(page_offset // page_limit) + 1} pages")
return all_widgets
except requests.exceptions.RequestException as e:
logging.info(f"Error during shared widgets auto-pagination: {e}")
return all_widgets
def fetch_system_wide_widgets(self):
"""Fetch all widgets owned by the fapi Drupal user (system-wide widgets)."""
fapi_target_id = os.getenv('FAPI_DRUPAL_TARGET_ID')
if not fapi_target_id:
logging.warning("FAPI_DRUPAL_TARGET_ID not configured")
return []
return self.fetch_widgets_by_uid(int(fapi_target_id))
def fetch_widget_by_id(self, widget_id: str):
"""Fetch a single widget by its Drupal node ID"""
endpoint = f"{DRUPAL_HOST}{DRUPAL_API}/node/widget/{widget_id}?include=uid"
try:
response = requests.get(endpoint)
if response.status_code == 200:
data = response.json()
return data.get('data')
elif response.status_code == 404:
logging.info(f"Widget not found: {widget_id}")
return None
else:
logging.info(f"Error fetching widget {widget_id}: {response.status_code} {response.text}")
return None
except requests.exceptions.RequestException as e:
logging.info(f"Error fetching widget {widget_id}: {e}")
return None
def create_widget(self, site, widget_data):
"""Parse a Drupal JSON:API widget node into a dict."""
body = widget_data.get('attributes', {}).get('body', {}).get('value', '{}')
body = json.loads(body)
owner_uuid = self.get_character_uid(widget_data)
site_id = ''
fapi_uuid = os.getenv('FAPI_DRUPAL_UUID')
is_system_wide = bool(fapi_uuid and owner_uuid == fapi_uuid)
if owner_uuid and not is_system_wide:
site_id = SiteConfigService().get_slug_by_drupal_uuid(owner_uuid) or ''
shared = widget_data.get('attributes', {}).get('field_shared2', True)
storage_id = widget_data.get('id', '')
mod_date = widget_data.get('attributes', {}).get('changed', '')
card = body.get('card', {})
return {
'storageId': storage_id,
'siteId': site_id,
'id': card.get('id', ''),
'name': card.get('title', ''),
'category': card.get('subtitle', ''),
'date': mod_date,
'enabled': card.get('enabled', False),
'hidden': card.get('hidden', False),
'config': body,
'shared': shared,
'systemWide': is_system_wide,
}
def create_characters(self, site, characters_data):
out_characters = []
for my_character_data in characters_data:
character = self.create_character(site, my_character_data)
out_characters.append(character)
return out_characters
# ── Site Config methods ──────────────────────────────────────────
def fetch_all_site_configs(self):
"""Fetch all site_config nodes with owner user included.
Returns (data_list, included_list) where included_list contains
the user entities needed to derive drupal_uid and drupal_target_id.
"""
all_nodes = []
all_included = {}
page_offset = 0
page_limit = 50
try:
while True:
endpoint = (
f"{DRUPAL_HOST}{DRUPAL_API}/node/site_config"
f"?include=uid"
f"&page[limit]={page_limit}&page[offset]={page_offset}"
)
response = requests.get(endpoint)
if response.status_code != 200:
msg = f"Error fetching site_config page: {response.status_code} {response.text}"
logging.info(msg)
if page_offset == 0:
raise Exception(msg)
break
data = response.json()
page_nodes = data.get('data', [])
if not page_nodes:
break
all_nodes.extend(page_nodes)
for item in data.get('included', []):
all_included[item['id']] = item
if len(page_nodes) < page_limit:
break
page_offset += page_limit
logging.info(f"Fetched {len(all_nodes)} site_config nodes")
return all_nodes, list(all_included.values())
except requests.exceptions.RequestException as e:
logging.info(f"Error fetching site_config nodes: {e}")
raise
def parse_site_config_nodes(self, data_list, included_list):
"""Convert JSON:API site_config nodes + included users to SiteConfig dicts.
Returns a list of dicts ready to construct SiteConfig objects (caller adds
api_host, drupal_host, drupal_api from env).
"""
from ..appTypes import SiteConfig
users_by_id = {u['id']: u for u in included_list}
api_host = os.getenv('API_HOST', 'http://localhost:8000/api/v1')
drupal_host = os.getenv('DRUPAL_HOST', 'http://drupal.test')
drupal_api = os.getenv('DRUPAL_API', '/jjknkk/jptb69dhj027ooae/jsonapi')
configs = []
for node in data_list:
attrs = node.get('attributes', {})
uid_data = node.get('relationships', {}).get('uid', {}).get('data', {})
user_uuid = uid_data.get('id')
drupal_target_id = uid_data.get('meta', {}).get('drupal_internal__target_id')
configs.append(SiteConfig(
cb_host=attrs.get('field_cb_host', ''),
wp_host=attrs.get('field_wp_host', ''),
wp_dir=attrs.get('field_wp_dir', ''),
slug=attrs.get('field_slug', ''),
api_host=api_host,
drupal_host=drupal_host,
drupal_api=drupal_api,
drupal_uid=user_uuid,
drupal_target_id=drupal_target_id,
drupal_node_id=node.get('id'),
))
return configs
async def create_site_config_node(self, session, csrf_token, slug, cb_host, wp_host, wp_dir, owner_uuid):
"""Create a site_config node in Drupal. Returns the created node data."""
endpoint = f"{DRUPAL_HOST}{DRUPAL_API}/node/site_config"
payload = {
"data": {
"type": "node--site_config",
"attributes": {
"title": slug,
"field_slug": slug,
"field_cb_host": cb_host,
"field_wp_host": wp_host,
"field_wp_dir": wp_dir,
},
"relationships": {
"uid": {
"data": {
"type": "user--user",
"id": owner_uuid,
}
}
}
}
}
response = session.post(
endpoint,
json=payload,
headers={
'Content-Type': 'application/vnd.api+json',
'Accept': 'application/vnd.api+json',
'X-CSRF-Token': csrf_token,
}
)
if response.status_code not in (200, 201):
logging.error(f"Failed to create site_config node: {response.status_code} {response.text}")
raise Exception(f"Drupal create failed: {response.status_code}")
return response.json().get('data')
async def delete_site_config_node(self, session, csrf_token, node_uuid):
"""Delete a site_config node by UUID."""
endpoint = f"{DRUPAL_HOST}{DRUPAL_API}/node/site_config/{node_uuid}"
response = session.delete(
endpoint,
headers={
'Accept': 'application/vnd.api+json',
'X-CSRF-Token': csrf_token,
}
)
if response.status_code not in (200, 204):
logging.error(f"Failed to delete site_config node: {response.status_code} {response.text}")
raise Exception(f"Drupal delete failed: {response.status_code}")
return True
def fetch_site_config_by_slug(self, slug):
"""Fetch a single site_config node by field_slug. Returns the node data or None."""
endpoint = (
f"{DRUPAL_HOST}{DRUPAL_API}/node/site_config"
f"?filter[field_slug]={slug}&include=uid"
)
try:
response = requests.get(endpoint)
if response.status_code == 200:
nodes = response.json().get('data', [])
if nodes:
return nodes[0]
return None
except requests.exceptions.RequestException as e:
logging.info(f"Error fetching site_config by slug '{slug}': {e}")
return None
def fetch_drupal_user_by_name(self, username, session=None):
"""Look up a Drupal user by username. Returns (uuid, target_id) or (None, None).
JSON:API filters on user/user require an authenticated session — pass one
if available, otherwise anonymous queries silently return an empty list.
"""
endpoint = f"{DRUPAL_HOST}{DRUPAL_API}/user/user?filter[name]={username}"
try:
response = (session or requests).get(endpoint)
if response.status_code == 200:
users = response.json().get('data', [])
if users:
user = users[0]
uuid = user.get('id')
target_id = user.get('attributes', {}).get('drupal_internal__uid')
return uuid, target_id
return None, None
except requests.exceptions.RequestException as e:
logging.info(f"Error fetching Drupal user '{username}': {e}")
return None, None