4453 lines
207 KiB
Python
4453 lines
207 KiB
Python
from __future__ import annotations
|
||
|
||
from typing import Any, Dict, List, Optional, Callable, Awaitable
|
||
from urllib.parse import urljoin
|
||
import json
|
||
import re
|
||
import asyncio
|
||
import time
|
||
import hashlib
|
||
from collections import deque
|
||
from agentui.providers.http_client import build_client
|
||
from agentui.common.vendors import detect_vendor
|
||
from agentui.providers.adapters.registry import get_adapter, default_base_url_for as _adapter_default_base_url_for
|
||
from agentui.pipeline.templating import (
|
||
_OUT_MACRO_RE,
|
||
_VAR_MACRO_RE,
|
||
_PROMPT_MACRO_RE,
|
||
_OUT_SHORT_RE,
|
||
_BRACES_RE,
|
||
_split_path,
|
||
_get_by_path,
|
||
_stringify_for_template,
|
||
_deep_find_text,
|
||
_best_text_from_outputs,
|
||
render_template_simple,
|
||
eval_condition_expr,
|
||
)
|
||
from agentui.pipeline.storage import load_var_store, save_var_store, clear_var_store
|
||
from agentui.common.cancel import is_cancelled, clear_cancel, get_cancel_mode
|
||
|
||
# HTTP request registry for manual resend feature (store original untrimmed bodies)
|
||
from collections import OrderedDict
|
||
|
||
_HTTP_REQ_REGISTRY_MAX = 200
|
||
_HTTP_REQ_REGISTRY = OrderedDict()
|
||
|
||
def register_http_request(req_id: str, info: Dict[str, Any]) -> None:
|
||
try:
|
||
rid = str(req_id or "")
|
||
if not rid:
|
||
return
|
||
# overwrite if exists to refresh order
|
||
if rid in _HTTP_REQ_REGISTRY:
|
||
try:
|
||
_HTTP_REQ_REGISTRY.pop(rid, None)
|
||
except Exception:
|
||
pass
|
||
_HTTP_REQ_REGISTRY[rid] = info or {}
|
||
# trim oldest beyond cap
|
||
while len(_HTTP_REQ_REGISTRY) > _HTTP_REQ_REGISTRY_MAX:
|
||
try:
|
||
_HTTP_REQ_REGISTRY.popitem(last=False)
|
||
except Exception:
|
||
break
|
||
except Exception:
|
||
pass
|
||
|
||
def get_http_request(req_id: str) -> Optional[Dict[str, Any]]:
|
||
try:
|
||
rid = str(req_id or "")
|
||
if not rid:
|
||
return None
|
||
v = _HTTP_REQ_REGISTRY.get(rid)
|
||
# return shallow copy to avoid mutation
|
||
return dict(v) if isinstance(v, dict) else None
|
||
except Exception:
|
||
return None
|
||
|
||
# --- Global helpers: robust auto-decompression for logging/JSON parsing ----------
|
||
import gzip
|
||
import zlib
|
||
|
||
def _decode_content_bytes(data: bytes, enc_header: str) -> Optional[bytes]:
|
||
"""
|
||
Try to decompress raw bytes according to Content-Encoding.
|
||
Returns decompressed bytes or None if unable/not needed.
|
||
"""
|
||
if not data:
|
||
return b""
|
||
enc = (enc_header or "").lower()
|
||
try:
|
||
if "br" in enc:
|
||
# Try brotli libs
|
||
brotli = None
|
||
try:
|
||
import brotlicffi as brotli # type: ignore
|
||
except Exception:
|
||
try:
|
||
import brotli # type: ignore
|
||
except Exception:
|
||
brotli = None
|
||
if brotli is not None:
|
||
try:
|
||
return brotli.decompress(data) # type: ignore[attr-defined]
|
||
except Exception:
|
||
try:
|
||
# Some packages expose .decompress as brotli.decompress
|
||
return brotli.decompress(data) # type: ignore
|
||
except Exception:
|
||
return None
|
||
return None
|
||
if "gzip" in enc:
|
||
try:
|
||
return gzip.decompress(data)
|
||
except Exception:
|
||
return None
|
||
if "deflate" in enc:
|
||
try:
|
||
return zlib.decompress(data)
|
||
except Exception:
|
||
# Some servers send raw deflate without zlib header
|
||
try:
|
||
return zlib.decompress(data, -zlib.MAX_WBITS)
|
||
except Exception:
|
||
return None
|
||
except Exception:
|
||
return None
|
||
return None
|
||
|
||
def _safe_response_text(resp) -> str:
|
||
"""
|
||
Return response text with best-effort auto-decompression (br/gzip/deflate) for logging.
|
||
Falls back to resp.text on failure.
|
||
"""
|
||
try:
|
||
enc = (resp.headers.get("content-encoding") or "").lower()
|
||
if enc:
|
||
dec = _decode_content_bytes(resp.content, enc)
|
||
if dec is not None:
|
||
try:
|
||
charset = resp.encoding or "utf-8"
|
||
return dec.decode(charset, errors="replace")
|
||
except Exception:
|
||
return dec.decode("utf-8", errors="replace")
|
||
# If no encoding or not handled — rely on httpx auto-decoding for gzip/deflate
|
||
return resp.text
|
||
except Exception:
|
||
try:
|
||
return resp.text
|
||
except Exception:
|
||
return "<resp.text decode error>"
|
||
|
||
def _safe_response_json(resp) -> Any:
|
||
"""
|
||
Return parsed JSON with auto-decompression fallback.
|
||
"""
|
||
try:
|
||
return resp.json()
|
||
except Exception:
|
||
t = _safe_response_text(resp)
|
||
try:
|
||
return json.loads(t)
|
||
except Exception:
|
||
return {"error": "Failed to decode JSON from upstream", "text": t}
|
||
|
||
|
||
# --- Cooperative cancel/abort helper for in-flight awaits (HTTP, etc.) ---------
|
||
async def _await_coro_with_cancel(coro: Awaitable[Any], pipeline_id: str, poll_interval: float = 0.1) -> Any:
|
||
"""
|
||
Await 'coro' while polling manual cancel flag for the pipeline.
|
||
|
||
Behavior:
|
||
- If cancel mode is 'graceful' -> do NOT interrupt 'coro'; we just continue waiting
|
||
and upper layers will stop before scheduling new work.
|
||
- If cancel mode is 'abort' -> cancel in-flight task immediately and raise ExecutionError.
|
||
|
||
Returns the result of 'coro' or raises ExecutionError on abort.
|
||
"""
|
||
task = asyncio.create_task(coro)
|
||
while True:
|
||
try:
|
||
# Shield from timeout cancellation; we only use timeout to poll flags.
|
||
return await asyncio.wait_for(asyncio.shield(task), timeout=poll_interval)
|
||
except asyncio.TimeoutError:
|
||
# Not done yet — fall through to polling
|
||
pass
|
||
except asyncio.CancelledError:
|
||
# Map in-flight cancellation to ExecutionError on abort; otherwise re-raise
|
||
try:
|
||
mode = get_cancel_mode(pipeline_id)
|
||
except Exception:
|
||
mode = "abort"
|
||
if mode == "abort":
|
||
raise ExecutionError("Cancelled by user (abort)")
|
||
raise
|
||
# Poll cancel flag
|
||
try:
|
||
if is_cancelled(pipeline_id):
|
||
mode = get_cancel_mode(pipeline_id)
|
||
if mode == "abort":
|
||
try:
|
||
if not task.done():
|
||
task.cancel()
|
||
await task
|
||
except asyncio.CancelledError:
|
||
raise ExecutionError("Cancelled by user (abort)")
|
||
except Exception as exc: # noqa: BLE001
|
||
raise ExecutionError(f"Cancelled by user (abort): {exc}")
|
||
# graceful: do not interrupt; keep waiting
|
||
except Exception:
|
||
# Be defensive: ignore polling failures and keep waiting
|
||
pass
|
||
|
||
|
||
# --- Helpers: sanitize base64/data URLs in JSON for logging (Burp-like) ----------
|
||
def _is_b64ish_string(s: str) -> bool:
|
||
try:
|
||
if not isinstance(s, str) or len(s) < 128:
|
||
return False
|
||
return bool(re.fullmatch(r"[A-Za-z0-9+/=\r\n]+", s))
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
_DATA_URL_RE = re.compile(r"(?is)^data:([^;]+);base64,([A-Za-z0-9+/=\r\n]+)$")
|
||
|
||
|
||
def _sanitize_b64_for_log(obj: Any, max_len: int = 180) -> Any:
|
||
"""
|
||
Recursively walk JSON-like obj and trim long base64 strings and data URLs for logging only.
|
||
- If key name contains 'data' (case-insensitive) and value looks like base64 → trim
|
||
- If value is data URL (data:mime;base64,...) → trim base64 part
|
||
Structure is preserved. Only strings are shortened.
|
||
"""
|
||
def trim_str(val: str, key_hint: Optional[str] = None) -> str:
|
||
# data URL?
|
||
m = _DATA_URL_RE.match(val or "")
|
||
if m:
|
||
mime, b64 = m.group(1), m.group(2)
|
||
if len(b64) > max_len:
|
||
return f"data:{mime};base64,{b64[:max_len]}... (trimmed {len(b64)-max_len})"
|
||
return val
|
||
# plain base64 by key hint
|
||
if key_hint and "data" in key_hint.lower() and _is_b64ish_string(val) and len(val) > max_len:
|
||
return val[:max_len] + f"... (trimmed {len(val)-max_len})"
|
||
return val
|
||
|
||
if isinstance(obj, dict):
|
||
out: Dict[str, Any] = {}
|
||
for k, v in obj.items():
|
||
if isinstance(v, str):
|
||
out[k] = trim_str(v, str(k))
|
||
else:
|
||
out[k] = _sanitize_b64_for_log(v, max_len)
|
||
return out
|
||
if isinstance(obj, list):
|
||
return [_sanitize_b64_for_log(v, max_len) for v in obj]
|
||
return obj
|
||
|
||
|
||
def _sanitize_json_string_for_log(s: str, max_len: int = 180) -> str:
|
||
"""
|
||
If 's' is a JSON string, sanitize base64/data URLs inside and dump back.
|
||
Otherwise return original string.
|
||
"""
|
||
try:
|
||
j = json.loads(s)
|
||
j2 = _sanitize_b64_for_log(j, max_len=max_len)
|
||
return json.dumps(j2, ensure_ascii=False, indent=2)
|
||
except Exception:
|
||
return s
|
||
|
||
|
||
# --- Templating helpers are imported from agentui.pipeline.templating ---
|
||
|
||
# moved to agentui.pipeline.templating
|
||
|
||
# moved to agentui.pipeline.templating
|
||
|
||
# moved to agentui.pipeline.templating
|
||
|
||
# moved to agentui.pipeline.templating
|
||
|
||
def _extract_out_node_id_from_ref(s: Any) -> Optional[str]:
|
||
"""
|
||
Извлекает node_id из строки с макросом [[OUT:nodeId(.path)*]].
|
||
Возвращает None, если макрос не найден.
|
||
"""
|
||
if not isinstance(s, str):
|
||
return None
|
||
m = _OUT_MACRO_RE.search(s)
|
||
if not m:
|
||
return None
|
||
body = m.group(1).strip()
|
||
node_id = body.split(".", 1)[0].strip()
|
||
return node_id or None
|
||
|
||
|
||
def _resolve_in_value(source: Any, context: Dict[str, Any], values: Dict[str, Dict[str, Any]]) -> Any:
|
||
"""
|
||
Разрешает входные связи/макросы в значение для inputs:
|
||
- Нестроковые значения возвращаются как есть
|
||
- "macro:path" → берёт значение из context по точечному пути
|
||
- "[[VAR:path]]" → берёт значение из context
|
||
- "[[OUT:nodeId(.path)*]]" → берёт из уже вычисленных выходов ноды
|
||
- "nodeId(.path)*" → ссылка на выходы ноды
|
||
- Если передан список ссылок — вернёт список разрешённых значений
|
||
- Иначе пытается взять из context по пути; если не найдено, оставляет исходную строку
|
||
"""
|
||
# Поддержка массивов ссылок (для multi-depends или будущих списковых входов)
|
||
if isinstance(source, list):
|
||
return [_resolve_in_value(s, context, values) for s in source]
|
||
if not isinstance(source, str):
|
||
return source
|
||
s = source.strip()
|
||
|
||
# macro:path
|
||
if s.lower().startswith("macro:"):
|
||
path = s.split(":", 1)[1].strip()
|
||
return _get_by_path(context, path)
|
||
|
||
# [[VAR: path]]
|
||
m = _VAR_MACRO_RE.fullmatch(s)
|
||
if m:
|
||
path = m.group(1).strip()
|
||
return _get_by_path(context, path)
|
||
|
||
# [[OUT: nodeId(.path)*]]
|
||
m = _OUT_MACRO_RE.fullmatch(s)
|
||
if m:
|
||
body = m.group(1).strip()
|
||
if "." in body:
|
||
node_id, rest = body.split(".", 1)
|
||
node_val = values.get(node_id)
|
||
return _get_by_path(node_val, rest)
|
||
node_val = values.get(body)
|
||
return node_val
|
||
|
||
# "nodeId(.path)*"
|
||
if "." in s:
|
||
node_id, rest = s.split(".", 1)
|
||
if node_id in values:
|
||
return _get_by_path(values.get(node_id), rest)
|
||
if s in values:
|
||
return values.get(s)
|
||
|
||
# fallback: from context by dotted path or raw string
|
||
ctx_val = _get_by_path(context, s)
|
||
return ctx_val if ctx_val is not None else source
|
||
|
||
|
||
# moved to agentui.pipeline.templating
|
||
|
||
|
||
class ExecutionError(Exception):
|
||
pass
|
||
|
||
|
||
class Node:
|
||
type_name: str = "Base"
|
||
|
||
def __init__(self, node_id: str, config: Optional[Dict[str, Any]] = None) -> None:
|
||
self.node_id = node_id
|
||
self.config = config or {}
|
||
|
||
async def run(self, inputs: Dict[str, Any], context: Dict[str, Any]) -> Dict[str, Any]: # noqa: D401
|
||
"""Execute node with inputs and context. Return dict of outputs."""
|
||
raise NotImplementedError
|
||
|
||
# Регистрация поддерживаемых типов нод (минимальный набор)
|
||
NODE_REGISTRY: Dict[str, Any] = {}
|
||
|
||
|
||
class PipelineExecutor:
|
||
def __init__(self, pipeline: Dict[str, Any]) -> None:
|
||
self.pipeline = pipeline
|
||
self.nodes_by_id: Dict[str, Node] = {}
|
||
for n in pipeline.get("nodes", []):
|
||
node_cls = NODE_REGISTRY.get(n.get("type"))
|
||
if not node_cls:
|
||
raise ExecutionError(f"Unknown node type: {n.get('type')}")
|
||
self.nodes_by_id[n["id"]] = node_cls(n["id"], n.get("config", {}))
|
||
|
||
# Настройки режима исполнения (с дефолтами из default_pipeline)
|
||
try:
|
||
self.loop_mode = str(self.pipeline.get("loop_mode", "dag")).lower().strip() or "dag"
|
||
except Exception:
|
||
self.loop_mode = "dag"
|
||
try:
|
||
self.loop_max_iters = int(self.pipeline.get("loop_max_iters", 1000))
|
||
except Exception:
|
||
self.loop_max_iters = 1000
|
||
try:
|
||
self.loop_time_budget_ms = int(self.pipeline.get("loop_time_budget_ms", 10000))
|
||
except Exception:
|
||
self.loop_time_budget_ms = 10000
|
||
|
||
# Идентификатор пайплайна и политика очистки стора переменных
|
||
try:
|
||
self.pipeline_id = str(self.pipeline.get("id", "pipeline_editor")) or "pipeline_editor"
|
||
except Exception:
|
||
self.pipeline_id = "pipeline_editor"
|
||
try:
|
||
self.clear_var_store = bool(self.pipeline.get("clear_var_store", True))
|
||
except Exception:
|
||
self.clear_var_store = True
|
||
|
||
# В памяти держим актуальный STORE (доступен в шаблонах через {{ store.* }} и [[STORE:*]])
|
||
self._store: Dict[str, Any] = {}
|
||
# Локальный журнал выполнения для человекочитаемого трейсинга
|
||
self._exec_log: List[str] = []
|
||
|
||
async def run(
|
||
self,
|
||
context: Dict[str, Any],
|
||
trace: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
Точка входа исполнителя. Переключает режим между:
|
||
- "dag": волновое исполнение с барьерами (исходное поведение)
|
||
- "iterative": итеративное исполнение с очередью и гейтами
|
||
"""
|
||
# Инициализация STORE по политике
|
||
try:
|
||
if self.clear_var_store:
|
||
clear_var_store(self.pipeline_id)
|
||
self._store = {}
|
||
else:
|
||
self._store = load_var_store(self.pipeline_id) or {}
|
||
except Exception:
|
||
self._store = {}
|
||
|
||
# Перед стартом прогона сбрасываем возможный «старый» флаг отмены
|
||
try:
|
||
clear_cancel(self.pipeline_id)
|
||
except Exception:
|
||
pass
|
||
|
||
mode = (self.loop_mode or "dag").lower()
|
||
if mode == "iterative":
|
||
res = await self._run_iterative(context, trace)
|
||
else:
|
||
res = await self._run_dag(context, trace)
|
||
|
||
# На всякий случай финальная запись стора (если были изменения)
|
||
try:
|
||
save_var_store(self.pipeline_id, self._store)
|
||
except Exception:
|
||
pass
|
||
return res
|
||
|
||
async def _exec_node(
|
||
self,
|
||
node_id: str,
|
||
ndef: Dict[str, Any],
|
||
values_snapshot: Dict[str, Any],
|
||
wave_num: int,
|
||
context: Dict[str, Any],
|
||
trace: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
|
||
user_vars_snapshot: Optional[Dict[str, Any]] = None,
|
||
) -> tuple[str, Dict[str, Any]]:
|
||
"""
|
||
Общий исполнитель ноды (используется и в DAG, и в iterative режимах)
|
||
Поведение идентично прежним локальным exec_one():
|
||
- подготовка контекста (OUT, vars, store, meta, _trace)
|
||
- разрешение inputs через макросы
|
||
- события trace node_start/node_done/node_error
|
||
- TRACE-печати начала/окончания с prev_text
|
||
"""
|
||
if not ndef:
|
||
raise ExecutionError(f"Node definition not found: {node_id}")
|
||
node = self.nodes_by_id[node_id]
|
||
|
||
# Сбор контекста
|
||
ctx = dict(context)
|
||
ctx["OUT"] = values_snapshot
|
||
try:
|
||
ctx["vars"] = dict(user_vars_snapshot or {})
|
||
except Exception:
|
||
ctx["vars"] = {}
|
||
# STORE доступен в шаблонах
|
||
try:
|
||
ctx["store"] = dict(self._store)
|
||
except Exception:
|
||
ctx["store"] = {}
|
||
|
||
# Pipeline meta (доступно нодам через context.meta)
|
||
try:
|
||
ctx["meta"] = {
|
||
"id": self.pipeline_id,
|
||
"loop_mode": self.loop_mode,
|
||
"loop_max_iters": self.loop_max_iters,
|
||
"loop_time_budget_ms": self.loop_time_budget_ms,
|
||
"clear_var_store": self.clear_var_store,
|
||
"http_timeout_sec": float(self.pipeline.get("http_timeout_sec", 60) or 60),
|
||
# v1: стратегия извлечения текста для [[OUTx]]
|
||
"text_extract_strategy": str(self.pipeline.get("text_extract_strategy", "auto") or "auto"),
|
||
"text_extract_json_path": str(self.pipeline.get("text_extract_json_path", "") or ""),
|
||
"text_join_sep": str(self.pipeline.get("text_join_sep", "\n") or "\n"),
|
||
# v2: коллекция пресетов парсинга для выбора в нодах
|
||
"text_extract_presets": list(self.pipeline.get("text_extract_presets", []) or []),
|
||
}
|
||
except Exception:
|
||
pass
|
||
# Прокидываем trace‑функцию в контекст, чтобы ноды могли посылать события (SSE лог)
|
||
try:
|
||
if trace is not None:
|
||
ctx["_trace"] = trace
|
||
except Exception:
|
||
pass
|
||
|
||
# Разворачиваем inputs
|
||
inputs: Dict[str, Any] = {}
|
||
for name, source in (ndef.get("in") or {}).items():
|
||
if isinstance(source, list):
|
||
inputs[name] = [_resolve_in_value(s, ctx, values_snapshot) for s in source]
|
||
else:
|
||
inputs[name] = _resolve_in_value(source, ctx, values_snapshot)
|
||
|
||
# Trace: node_start
|
||
if trace is not None:
|
||
try:
|
||
await trace({
|
||
"event": "node_start",
|
||
"node_id": ndef["id"],
|
||
"node_type": node.type_name,
|
||
"wave": wave_num,
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
|
||
started = time.perf_counter()
|
||
try:
|
||
print(f"TRACE start: {ndef['id']} ({node.type_name})")
|
||
except Exception:
|
||
pass
|
||
|
||
# Исполнение
|
||
try:
|
||
out = await node.run(inputs, ctx)
|
||
except BaseException as exc:
|
||
err = exc
|
||
try:
|
||
import asyncio as _asyncio
|
||
if isinstance(exc, _asyncio.CancelledError):
|
||
err = ExecutionError("Cancelled by user (abort)")
|
||
except Exception:
|
||
pass
|
||
if trace is not None:
|
||
try:
|
||
await trace({
|
||
"event": "node_error",
|
||
"node_id": ndef["id"],
|
||
"node_type": node.type_name,
|
||
"wave": wave_num,
|
||
"ts": int(time.time() * 1000),
|
||
"error": str(err),
|
||
})
|
||
except Exception:
|
||
pass
|
||
raise err
|
||
else:
|
||
dur_ms = int((time.perf_counter() - started) * 1000)
|
||
if trace is not None:
|
||
try:
|
||
await trace({
|
||
"event": "node_done",
|
||
"node_id": ndef["id"],
|
||
"node_type": node.type_name,
|
||
"wave": wave_num,
|
||
"ts": int(time.time() * 1000),
|
||
"duration_ms": dur_ms,
|
||
})
|
||
except Exception:
|
||
pass
|
||
try:
|
||
prev_text = ""
|
||
try:
|
||
from agentui.pipeline.templating import _best_text_from_outputs as _bt # type: ignore
|
||
if isinstance(out, dict):
|
||
prev_text = _bt(out) or ""
|
||
except Exception:
|
||
prev_text = ""
|
||
if isinstance(prev_text, str) and len(prev_text) > 200:
|
||
prev_text = prev_text[:200]
|
||
print(f"TRACE done: {ndef['id']} ({node.type_name}) dur={dur_ms}ms text={prev_text!r}")
|
||
except Exception:
|
||
pass
|
||
return node_id, out
|
||
async def _run_dag(
|
||
self,
|
||
context: Dict[str, Any],
|
||
trace: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
Исходная волновая модель (DAG) — без изменений поведения.
|
||
"""
|
||
nodes: List[Dict[str, Any]] = list(self.pipeline.get("nodes", []))
|
||
id_set = set(self.nodes_by_id.keys())
|
||
|
||
# Собираем зависимости: node_id -> set(parent_ids), и обратные связи dependents
|
||
deps_map: Dict[str, set] = {n["id"]: set() for n in nodes}
|
||
dependents: Dict[str, set] = {n["id"]: set() for n in nodes}
|
||
# Гейты для ветвлений: child_id -> List[(parent_id, gate_name)] (gate_name: "true"/"false")
|
||
gate_deps: Dict[str, List[tuple[str, str]]] = {n["id"]: [] for n in nodes}
|
||
|
||
for n in nodes:
|
||
nid = n["id"]
|
||
for _, source in (n.get("in") or {}).items():
|
||
# Разворачиваем массивы ссылок (multi-depends)
|
||
sources = source if isinstance(source, list) else [source]
|
||
for src in sources:
|
||
if not isinstance(src, str):
|
||
# Нестрочные значения и массивы констант — зависимостей нет
|
||
continue
|
||
if src.startswith("macro:"):
|
||
# Макросы берутся из контекста, без зависимостей
|
||
continue
|
||
# [[VAR:...]] — макрос из контекста, зависимостей нет
|
||
if re.fullmatch(r"\[\[\s*VAR\s*[:\s]\s*[^\]]+\s*\]\]", src.strip()):
|
||
continue
|
||
# [[OUT:nodeId(.key)*]] — зависимость от указанной ноды
|
||
out_ref_node = _extract_out_node_id_from_ref(src)
|
||
if out_ref_node and out_ref_node in id_set:
|
||
deps_map[nid].add(out_ref_node)
|
||
dependents[out_ref_node].add(nid)
|
||
continue
|
||
# Ссылки вида "node.outKey" или "node"
|
||
src_id = src.split(".", 1)[0] if "." in src else src
|
||
if src_id in id_set:
|
||
# Если указали конкретный outKey (например, nIf.true / nIf.false),
|
||
# трактуем это ТОЛЬКО как гейт (НЕ как топологическую зависимость), чтобы избежать дедлоков.
|
||
is_gate = False
|
||
if "." in src:
|
||
try:
|
||
_, out_name = src.split(".", 1)
|
||
on = str(out_name).strip().lower()
|
||
if on in {"true", "false"}:
|
||
gate_deps[nid].append((src_id, on))
|
||
# Smart mode v2: If-node may act as a real parent ONLY when the child has no real parents yet
|
||
# AND adding this parent won't create an immediate cycle (i.e. If-node does not depend on this child).
|
||
if len(deps_map[nid]) == 0:
|
||
dep_on_child = False
|
||
try:
|
||
src_def = next((x for x in nodes if str(x.get("id")) == str(src_id)), None)
|
||
if src_def and isinstance(src_def.get("in"), dict):
|
||
for _, s2 in (src_def.get("in") or {}).items():
|
||
arr = s2 if isinstance(s2, list) else [s2]
|
||
for it2 in arr:
|
||
if isinstance(it2, str):
|
||
base = it2.split(".", 1)[0] if "." in it2 else it2
|
||
if str(base) == str(nid):
|
||
dep_on_child = True
|
||
break
|
||
if dep_on_child:
|
||
break
|
||
except Exception:
|
||
dep_on_child = False
|
||
if not dep_on_child:
|
||
deps_map[nid].add(src_id)
|
||
dependents[src_id].add(nid)
|
||
is_gate = True
|
||
except Exception:
|
||
is_gate = False
|
||
if not is_gate:
|
||
deps_map[nid].add(src_id)
|
||
dependents[src_id].add(nid)
|
||
|
||
# Входящие степени и первая волна
|
||
in_degree: Dict[str, int] = {nid: len(deps) for nid, deps in deps_map.items()}
|
||
ready: List[str] = [nid for nid, deg in in_degree.items() if deg == 0]
|
||
|
||
processed: List[str] = []
|
||
values: Dict[str, Dict[str, Any]] = {}
|
||
last_result: Dict[str, Any] = {}
|
||
last_node_id: Optional[str] = None
|
||
node_def_by_id: Dict[str, Dict[str, Any]] = {n["id"]: n for n in nodes}
|
||
user_vars: Dict[str, Any] = {}
|
||
|
||
try:
|
||
parallel_limit = int(self.pipeline.get("parallel_limit", 8))
|
||
except Exception:
|
||
parallel_limit = 8
|
||
if parallel_limit <= 0:
|
||
parallel_limit = 1
|
||
|
||
async def exec_one(node_id: str, values_snapshot: Dict[str, Any], wave_num: int) -> tuple[str, Dict[str, Any]]:
|
||
ndef = node_def_by_id.get(node_id)
|
||
return await self._exec_node(node_id, ndef, values_snapshot, wave_num, context, trace, user_vars)
|
||
|
||
wave_idx = 0
|
||
while ready:
|
||
# Ручная отмена исполнения (DAG)
|
||
try:
|
||
if is_cancelled(self.pipeline_id):
|
||
if trace is not None:
|
||
try:
|
||
await trace({
|
||
"event": "cancelled",
|
||
"node_id": "",
|
||
"node_type": "Pipeline",
|
||
"wave": wave_idx,
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
# Снимок и финальный EXEC TRACE
|
||
try:
|
||
self._commit_snapshot(context, values, last_node_id or "")
|
||
except Exception:
|
||
pass
|
||
try:
|
||
summary = " -> ".join(self._exec_log) if getattr(self, "_exec_log", None) else ""
|
||
if summary and isinstance(self._store.get("snapshot"), dict):
|
||
self._store["snapshot"]["EXEC_TRACE"] = summary
|
||
save_var_store(self.pipeline_id, self._store)
|
||
except Exception:
|
||
pass
|
||
return last_result
|
||
except Exception:
|
||
pass
|
||
|
||
wave_nodes = list(ready)
|
||
ready = []
|
||
wave_results: Dict[str, Dict[str, Any]] = {}
|
||
values_snapshot = dict(values)
|
||
|
||
for i in range(0, len(wave_nodes), parallel_limit):
|
||
chunk = wave_nodes[i : i + parallel_limit]
|
||
results = await asyncio.gather(
|
||
*(exec_one(nid, values_snapshot, wave_idx) for nid in chunk),
|
||
return_exceptions=False,
|
||
)
|
||
for nid, out in results:
|
||
wave_results[nid] = out
|
||
last_result = out
|
||
last_node_id = nid
|
||
try:
|
||
ndef = node_def_by_id.get(nid) or {}
|
||
ntype = ndef.get("type", "")
|
||
if ntype == "If":
|
||
expr = str((ndef.get("config") or {}).get("expr", ""))
|
||
resb = False
|
||
try:
|
||
if isinstance(out, dict):
|
||
resb = bool(out.get("result"))
|
||
except Exception:
|
||
resb = False
|
||
self._exec_log.append(f"If (#{nid}) {expr} => {str(resb).lower()}")
|
||
else:
|
||
self._exec_log.append(f"{nid}({ntype})")
|
||
except Exception:
|
||
pass
|
||
|
||
values.update(wave_results)
|
||
processed.extend(wave_nodes)
|
||
|
||
try:
|
||
for _nid, out in wave_results.items():
|
||
if isinstance(out, dict):
|
||
v = out.get("vars")
|
||
if isinstance(v, dict):
|
||
user_vars.update(v)
|
||
except Exception:
|
||
pass
|
||
|
||
# Обновляем и сохраняем STORE, если были новые переменные из SetVars/прочих нод
|
||
try:
|
||
merged: Dict[str, Any] = {}
|
||
for _nid, out in wave_results.items():
|
||
if isinstance(out, dict) and isinstance(out.get("vars"), dict):
|
||
merged.update(out.get("vars") or {})
|
||
if merged:
|
||
self._store.update(merged)
|
||
try:
|
||
save_var_store(self.pipeline_id, self._store)
|
||
except Exception:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
|
||
for done in wave_nodes:
|
||
for child in dependents.get(done, ()):
|
||
in_degree[child] -= 1
|
||
next_ready_candidates = [nid for nid, deg in in_degree.items() if deg == 0 and nid not in processed and nid not in wave_nodes]
|
||
|
||
def _gates_ok(child_id: str) -> bool:
|
||
pairs = gate_deps.get(child_id) or []
|
||
if not pairs:
|
||
return True
|
||
missing_seen = False
|
||
for pid, gate in pairs:
|
||
v = values.get(pid)
|
||
if v is None:
|
||
# значение гейта ещё не известно
|
||
missing_seen = True
|
||
continue
|
||
try:
|
||
flag = bool(isinstance(v, dict) and v.get(gate))
|
||
except Exception:
|
||
flag = False
|
||
if not flag:
|
||
return False
|
||
# Семантика "missing gate = pass" как в iterative-режиме:
|
||
# если флаги гейта ещё не известны, но у ребёнка есть реальные родители — допускаем первый запуск.
|
||
# если реальных родителей нет (только гейты) — ждём появления значения гейта.
|
||
if missing_seen:
|
||
real_parents = deps_map.get(child_id) or set()
|
||
if len(real_parents) == 0:
|
||
return False
|
||
return True
|
||
|
||
ready = [nid for nid in next_ready_candidates if _gates_ok(nid)]
|
||
wave_idx += 1
|
||
|
||
if len(processed) != len(nodes):
|
||
remaining = [n["id"] for n in nodes if n["id"] not in processed]
|
||
raise ExecutionError(f"Cycle detected or unresolved dependencies among nodes: {remaining}")
|
||
|
||
# Persist snapshot of macro context and outputs into STORE
|
||
try:
|
||
self._commit_snapshot(context, values, last_node_id or "")
|
||
except Exception:
|
||
pass
|
||
|
||
# Построение и сохранение человекочитаемого EXECUTION TRACE
|
||
try:
|
||
summary = " -> ".join(self._exec_log) if getattr(self, "_exec_log", None) else ""
|
||
if summary:
|
||
try:
|
||
print("===== EXECUTION TRACE =====")
|
||
print(summary)
|
||
print("===== END TRACE =====")
|
||
except Exception:
|
||
pass
|
||
try:
|
||
if isinstance(self._store.get("snapshot"), dict):
|
||
self._store["snapshot"]["EXEC_TRACE"] = summary
|
||
save_var_store(self.pipeline_id, self._store)
|
||
except Exception:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
|
||
return last_result
|
||
|
||
async def _run_iterative(
|
||
self,
|
||
context: Dict[str, Any],
|
||
trace: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
Итеративное исполнение:
|
||
- Очередь готовых задач
|
||
- Повторные пуски допустимы: ребёнок ставится в очередь, когда какой-либо из его родителей обновился после последнего запуска ребёнка
|
||
- Гейты If.true/If.false фильтруют постановку в очередь
|
||
- Ограничения: loop_max_iters, loop_time_budget_ms
|
||
"""
|
||
nodes: List[Dict[str, Any]] = list(self.pipeline.get("nodes", []))
|
||
id_set = set(self.nodes_by_id.keys())
|
||
|
||
# Строим зависимости и гейты (как в DAG)
|
||
deps_map: Dict[str, set] = {n["id"]: set() for n in nodes}
|
||
dependents: Dict[str, set] = {n["id"]: set() for n in nodes}
|
||
gate_deps: Dict[str, List[tuple[str, str]]] = {n["id"]: [] for n in nodes}
|
||
# Обратная карта для гейтов: кто «слушает» изменения конкретного родителя по гейту
|
||
gate_dependents: Dict[str, set] = {n["id"]: set() for n in nodes}
|
||
|
||
for n in nodes:
|
||
nid = n["id"]
|
||
for _, source in (n.get("in") or {}).items():
|
||
sources = source if isinstance(source, list) else [source]
|
||
for src in sources:
|
||
if not isinstance(src, str):
|
||
continue
|
||
if src.startswith("macro:"):
|
||
continue
|
||
if re.fullmatch(r"\[\[\s*VAR\s*[:\s]\s*[^\]]+\s*\]\]", src.strip()):
|
||
continue
|
||
out_ref_node = _extract_out_node_id_from_ref(src)
|
||
if out_ref_node and out_ref_node in id_set:
|
||
# [[OUT:nodeId(.key)*]] — это РЕАЛЬНАЯ топологическая зависимость
|
||
deps_map[nid].add(out_ref_node)
|
||
dependents[out_ref_node].add(nid)
|
||
continue
|
||
src_id = src.split(".", 1)[0] if "." in src else src
|
||
if src_id in id_set:
|
||
# "node.true/false" — ТОЛЬКО гейт (без топологической зависимости)
|
||
is_gate = False
|
||
if "." in src:
|
||
try:
|
||
_, out_name = src.split(".", 1)
|
||
on = str(out_name).strip().lower()
|
||
if on in {"true", "false"}:
|
||
gate_deps[nid].append((src_id, on))
|
||
# Регистрируем обратную зависимость по гейту,
|
||
# чтобы будить ребёнка при изменении флага у родителя
|
||
gate_dependents[src_id].add(nid)
|
||
# Smart mode v2: делаем If-узел реальным родителем ТОЛЬКО если
|
||
# у ребёнка ещё нет реальных родителей И это не создаёт цикл (If не зависит от ребёнка).
|
||
if len(deps_map[nid]) == 0:
|
||
dep_on_child = False
|
||
try:
|
||
src_def = next((x for x in nodes if str(x.get("id")) == str(src_id)), None)
|
||
if src_def and isinstance(src_def.get("in"), dict):
|
||
for _, s2 in (src_def.get("in") or {}).items():
|
||
arr = s2 if isinstance(s2, list) else [s2]
|
||
for it2 in arr:
|
||
if isinstance(it2, str):
|
||
base = it2.split(".", 1)[0] if "." in it2 else it2
|
||
if str(base) == str(nid):
|
||
dep_on_child = True
|
||
break
|
||
if dep_on_child:
|
||
break
|
||
except Exception:
|
||
dep_on_child = False
|
||
if not dep_on_child:
|
||
deps_map[nid].add(src_id)
|
||
dependents[src_id].add(nid)
|
||
is_gate = True
|
||
except Exception:
|
||
is_gate = False
|
||
if not is_gate:
|
||
deps_map[nid].add(src_id)
|
||
dependents[src_id].add(nid)
|
||
|
||
# Вспом. структуры состояния
|
||
values: Dict[str, Dict[str, Any]] = {}
|
||
last_result: Dict[str, Any] = {}
|
||
last_node_id: Optional[str] = None
|
||
user_vars: Dict[str, Any] = {}
|
||
node_def_by_id: Dict[str, Dict[str, Any]] = {n["id"]: n for n in nodes}
|
||
|
||
# Версии обновлений нод (инкремент при каждом коммите)
|
||
version: Dict[str, int] = {n["id"]: 0 for n in nodes}
|
||
# Когда ребёнок запускался в последний раз (номер глобального шага)
|
||
last_run_step: Dict[str, int] = {n["id"]: -1 for n in nodes}
|
||
# Глобальный счётчик шагов
|
||
step = 0
|
||
|
||
# Очередь готовых задач
|
||
q: deque[str] = deque()
|
||
|
||
# Начальные готовые — ноды без зависимостей
|
||
in_degree: Dict[str, int] = {nid: len(deps) for nid, deps in deps_map.items()}
|
||
for nid, deg in in_degree.items():
|
||
if deg == 0:
|
||
q.append(nid)
|
||
|
||
def gates_ok(child_id: str) -> bool:
|
||
pairs = gate_deps.get(child_id) or []
|
||
if not pairs:
|
||
return True
|
||
missing_seen = False
|
||
for pid, gate in pairs:
|
||
v = values.get(pid)
|
||
# Если значение гейта ещё не появилось — отмечаем и продолжаем проверку других гейтов
|
||
if v is None:
|
||
missing_seen = True
|
||
continue
|
||
try:
|
||
flag = bool(isinstance(v, dict) and v.get(gate))
|
||
except Exception:
|
||
flag = False
|
||
if not flag:
|
||
return False
|
||
# Если были отсутствующие значения гейтов, то:
|
||
# - у узла есть реальные (топологические) родители → не блокируем (первый запуск допустим)
|
||
# - у узла НЕТ реальных родителей (только гейты) → откладываем до появления значения гейта
|
||
if missing_seen:
|
||
real_parents = deps_map.get(child_id) or set()
|
||
if len(real_parents) == 0:
|
||
return False
|
||
return True
|
||
|
||
# Лимиты
|
||
total_runs = 0
|
||
t0 = time.perf_counter()
|
||
try:
|
||
parallel_limit = int(self.pipeline.get("parallel_limit", 8))
|
||
except Exception:
|
||
parallel_limit = 8
|
||
if parallel_limit <= 0:
|
||
parallel_limit = 1
|
||
|
||
async def exec_one(node_id: str, snapshot: Dict[str, Any], wave_num: int) -> tuple[str, Dict[str, Any]]:
|
||
ndef = node_def_by_id.get(node_id)
|
||
return await self._exec_node(node_id, ndef, snapshot, wave_num, context, trace, user_vars)
|
||
|
||
# Главный цикл
|
||
while q:
|
||
# Ручная отмена исполнения (iterative)
|
||
try:
|
||
if is_cancelled(self.pipeline_id):
|
||
if trace is not None:
|
||
try:
|
||
await trace({
|
||
"event": "cancelled",
|
||
"node_id": "",
|
||
"node_type": "Pipeline",
|
||
"wave": step,
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
# Снимок и финальный EXEC TRACE
|
||
try:
|
||
self._commit_snapshot(context, values, last_node_id or "")
|
||
except Exception:
|
||
pass
|
||
try:
|
||
summary = " -> ".join(self._exec_log) if getattr(self, "_exec_log", None) else ""
|
||
if summary and isinstance(self._store.get("snapshot"), dict):
|
||
self._store["snapshot"]["EXEC_TRACE"] = summary
|
||
save_var_store(self.pipeline_id, self._store)
|
||
except Exception:
|
||
pass
|
||
return last_result
|
||
except Exception:
|
||
pass
|
||
|
||
# Проверяем лимиты
|
||
if total_runs >= self.loop_max_iters:
|
||
raise ExecutionError(f"Iterative mode exceeded loop_max_iters={self.loop_max_iters}")
|
||
if (time.perf_counter() - t0) * 1000.0 > self.loop_time_budget_ms:
|
||
raise ExecutionError(f"Iterative mode exceeded loop_time_budget_ms={self.loop_time_budget_ms}")
|
||
|
||
# Собираем партию до parallel_limit, с учётом гейтов.
|
||
# ВАЖНО: защищаемся от «вечной карусели» в пределах одной партии.
|
||
# Обрабатываем не более исходной длины очереди за проход.
|
||
batch: List[str] = []
|
||
seen: set[str] = set()
|
||
spin_guard = len(q)
|
||
processed_in_pass = 0
|
||
while q and len(batch) < parallel_limit and processed_in_pass < spin_guard:
|
||
nid = q.popleft()
|
||
processed_in_pass += 1
|
||
if nid in seen:
|
||
# Уже добавлен в партию — пропускаем без возврата в очередь
|
||
continue
|
||
# Должны быть удовлетворены зависимости: или нет родителей, или все родители хотя бы раз исполнились
|
||
parents = deps_map.get(nid) or set()
|
||
deps_ok = all(version[p] > 0 or in_degree.get(nid, 0) == 0 for p in parents) or (len(parents) == 0)
|
||
if not deps_ok:
|
||
# отложим до лучших времён
|
||
q.append(nid)
|
||
continue
|
||
if not gates_ok(nid):
|
||
# гейты пока не открыты — попробуем позже
|
||
q.append(nid)
|
||
continue
|
||
batch.append(nid)
|
||
seen.add(nid)
|
||
|
||
if not batch:
|
||
# Очередь «застряла» — либо циклические ожидания без начальных значений, либо гейт не откроется.
|
||
# Безопасно выходим.
|
||
break
|
||
|
||
# Снимок на момент партии
|
||
snapshot = dict(values)
|
||
results = await asyncio.gather(*(exec_one(nid, snapshot, step) for nid in batch), return_exceptions=False)
|
||
|
||
# Коммитим результаты немедленно и продвигаем версии
|
||
produced_vars: Dict[str, Any] = {}
|
||
for nid, out in results:
|
||
values[nid] = out
|
||
version[nid] = version.get(nid, 0) + 1
|
||
last_result = out
|
||
last_node_id = nid
|
||
total_runs += 1
|
||
last_run_step[nid] = step
|
||
# Человекочитаемый журнал
|
||
try:
|
||
ndef = node_def_by_id.get(nid) or {}
|
||
ntype = ndef.get("type", "")
|
||
if ntype == "If":
|
||
expr = str((ndef.get("config") or {}).get("expr", ""))
|
||
resb = False
|
||
try:
|
||
if isinstance(out, dict):
|
||
resb = bool(out.get("result"))
|
||
except Exception:
|
||
resb = False
|
||
self._exec_log.append(f"If (#{nid}) {expr} => {str(resb).lower()}")
|
||
else:
|
||
self._exec_log.append(f"{nid}({ntype})")
|
||
except Exception:
|
||
pass
|
||
|
||
# Сбор пользовательских переменных
|
||
try:
|
||
if isinstance(out, dict) and isinstance(out.get("vars"), dict):
|
||
produced_vars.update(out.get("vars") or {})
|
||
except Exception:
|
||
pass
|
||
|
||
if produced_vars:
|
||
user_vars.update(produced_vars)
|
||
# Обновляем STORE и сохраняем на диск
|
||
try:
|
||
self._store.update(produced_vars)
|
||
save_var_store(self.pipeline_id, self._store)
|
||
except Exception:
|
||
pass
|
||
|
||
# Постановка потомков, у кого изменился хотя бы один родитель
|
||
for nid in batch:
|
||
# Топологические потомки (реальные зависимости): будим всегда
|
||
for child in dependents.get(nid, ()):
|
||
if last_run_step.get(child, -1) < step:
|
||
q.append(child)
|
||
|
||
# Потомки по гейтам: будим ТОЛЬКО выбранную ветку
|
||
# Логика: если у gchild есть пара gate_deps с текущим nid и нужным gate ('true'/'false'),
|
||
# то будим только если соответствующий флаг у родителя истинный. Иначе — не будим.
|
||
g_out = values.get(nid) or {}
|
||
g_children = gate_dependents.get(nid, ())
|
||
for gchild in g_children:
|
||
if last_run_step.get(gchild, -1) >= step:
|
||
continue
|
||
pairs = [p for p in (gate_deps.get(gchild) or []) if p and p[0] == nid]
|
||
if not pairs:
|
||
# gchild зарегистрирован как зависимый по гейту, но явной пары нет — пропускаем
|
||
continue
|
||
wake = False
|
||
for (_pid, gate_name) in pairs:
|
||
try:
|
||
want = str(gate_name or "").strip().lower()
|
||
if want in ("true", "false"):
|
||
if bool(g_out.get(want, False)):
|
||
wake = True
|
||
break
|
||
except Exception:
|
||
pass
|
||
if wake:
|
||
q.append(gchild)
|
||
try:
|
||
print(f"TRACE wake_by_gate: parent={nid} -> child={gchild}")
|
||
except Exception:
|
||
pass
|
||
|
||
# Ранний выход, если встретили Return среди выполненных
|
||
try:
|
||
for nid in batch:
|
||
ndef = node_def_by_id.get(nid) or {}
|
||
if (ndef.get("type") or "") == "Return":
|
||
# Зафиксируем, что именно Return завершил выполнение
|
||
try:
|
||
last_node_id = nid
|
||
last_result = values.get(nid) or last_result
|
||
except Exception:
|
||
pass
|
||
# Снимок стора до выхода
|
||
try:
|
||
self._commit_snapshot(context, values, last_node_id or "")
|
||
except Exception:
|
||
pass
|
||
# Финальный EXECUTION TRACE в лог и STORE
|
||
try:
|
||
summary = " -> ".join(self._exec_log) if getattr(self, "_exec_log", None) else ""
|
||
if summary:
|
||
try:
|
||
print("===== EXECUTION TRACE =====")
|
||
print(summary)
|
||
print("===== END TRACE =====")
|
||
except Exception:
|
||
pass
|
||
try:
|
||
if isinstance(self._store.get("snapshot"), dict):
|
||
self._store["snapshot"]["EXEC_TRACE"] = summary
|
||
save_var_store(self.pipeline_id, self._store)
|
||
except Exception:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
return last_result
|
||
except Exception:
|
||
pass
|
||
|
||
step += 1
|
||
|
||
# Финальный снапшот всего контекста и OUT в STORE.snapshot
|
||
try:
|
||
self._commit_snapshot(context, values, last_node_id or "")
|
||
except Exception:
|
||
pass
|
||
|
||
# Построение и сохранение человекочитаемого EXECUTION TRACE
|
||
try:
|
||
summary = " -> ".join(self._exec_log) if getattr(self, "_exec_log", None) else ""
|
||
if summary:
|
||
try:
|
||
print("===== EXECUTION TRACE =====")
|
||
print(summary)
|
||
print("===== END TRACE =====")
|
||
except Exception:
|
||
pass
|
||
try:
|
||
if isinstance(self._store.get("snapshot"), dict):
|
||
self._store["snapshot"]["EXEC_TRACE"] = summary
|
||
save_var_store(self.pipeline_id, self._store)
|
||
except Exception:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
|
||
return last_result
|
||
|
||
|
||
def _safe_preview(self, obj: Any, max_bytes: int = 262_144) -> Any:
|
||
"""
|
||
Отменяем тримминг: сохраняем объект полностью.
|
||
Важно: это может сделать STORE очень большим для тяжёлых ответов.
|
||
"""
|
||
return obj
|
||
|
||
def _commit_snapshot(self, context: Dict[str, Any], values: Dict[str, Any], last_node_id: str) -> None:
|
||
"""
|
||
Собирает STORE.snapshot:
|
||
- incoming.*, params.*, model, vendor_format, system
|
||
- OUT_TEXT.nX — строковая вытяжка (до 16КБ)
|
||
- OUT.nX — сырой JSON (с триммингом до ~256КБ через preview)
|
||
- Алиасы OUT1/OUT2/... — плоские строки (то же, что [[OUT1]])
|
||
- LAST_NODE — последний выполнившийся узел
|
||
"""
|
||
# OUT_TEXT / OUT
|
||
out_text: Dict[str, str] = {}
|
||
out_raw: Dict[str, Any] = {}
|
||
aliases: Dict[str, Any] = {}
|
||
|
||
for nid, out in (values or {}).items():
|
||
if out is None:
|
||
continue
|
||
try:
|
||
txt = _best_text_from_outputs(out) # type: ignore[name-defined]
|
||
except Exception:
|
||
txt = ""
|
||
if not isinstance(txt, str):
|
||
try:
|
||
txt = str(txt)
|
||
except Exception:
|
||
txt = ""
|
||
# Без тримминга текста — сохраняем полный text
|
||
out_text[nid] = txt
|
||
out_raw[nid] = self._safe_preview(out, 262_144)
|
||
m = re.match(r"^n(\d+)$", str(nid))
|
||
if m:
|
||
aliases[f"OUT{int(m.group(1))}"] = txt
|
||
|
||
snapshot: Dict[str, Any] = {
|
||
"incoming": self._safe_preview(context.get("incoming"), 262_144),
|
||
"params": self._safe_preview(context.get("params"), 65_536),
|
||
"model": context.get("model"),
|
||
"vendor_format": context.get("vendor_format"),
|
||
"system": context.get("system") or "",
|
||
"OUT": out_raw,
|
||
"OUT_TEXT": out_text,
|
||
"LAST_NODE": last_node_id or "",
|
||
**aliases,
|
||
}
|
||
|
||
# Сохраняем под ключом snapshot, пользовательские vars остаются как есть в корне STORE
|
||
try:
|
||
if not isinstance(self._store, dict):
|
||
self._store = {}
|
||
self._store["snapshot"] = snapshot
|
||
save_var_store(self.pipeline_id, self._store)
|
||
except Exception:
|
||
pass
|
||
|
||
class SetVarsNode(Node):
|
||
type_name = "SetVars"
|
||
|
||
def _normalize(self) -> List[Dict[str, Any]]:
|
||
raw = self.config.get("variables") or []
|
||
if not isinstance(raw, list):
|
||
return []
|
||
norm: List[Dict[str, Any]] = []
|
||
for i, b in enumerate(raw):
|
||
if not isinstance(b, dict):
|
||
continue
|
||
name = str(b.get("name", "")).strip()
|
||
mode = str(b.get("mode", "string")).lower().strip()
|
||
value = b.get("value", "")
|
||
try:
|
||
order = int(b.get("order")) if b.get("order") is not None else i
|
||
except Exception:
|
||
order = i
|
||
norm.append({
|
||
"id": b.get("id") or f"v{i}",
|
||
"name": name,
|
||
"mode": "expr" if mode == "expr" else "string",
|
||
"value": value,
|
||
"order": order,
|
||
})
|
||
return norm
|
||
|
||
def _safe_eval_expr(self, expr: str, context: Optional[Dict[str, Any]] = None, out_map: Optional[Dict[str, Any]] = None) -> Any:
|
||
"""
|
||
Безопасная оценка выражений для SetVars.
|
||
|
||
Поддержка:
|
||
- Литералы: числа/строки/bool/None, списки, кортежи, словари
|
||
- JSON‑литералы: true/false/null, объекты и массивы (парсятся как Python True/False/None, dict/list)
|
||
- Арифметика: + - * / // %, унарные +-
|
||
- Логика: and/or, сравнения (== != < <= > >=, цепочки)
|
||
- Безопасные функции:
|
||
rand(), randint(a,b), choice(list)
|
||
jp(value, path, join_sep="\n"), jp_text(value, path, join_sep="\n")
|
||
from_json(x)
|
||
file_b64(path), data_url(b64, mime), file_data_url(path, mime?)
|
||
|
||
Примечание: функции jp/jp_text/from_json принимают как обычные значения,
|
||
так и строки с макросами ([[...]] или {{ ... }}). Макросы разворачиваются с помощью render_template_simple.
|
||
"""
|
||
import ast
|
||
import operator as op
|
||
import random
|
||
import base64
|
||
import os
|
||
import mimetypes
|
||
from pathlib import Path
|
||
|
||
def _resolve_input(x: Any) -> Any:
|
||
# Если уже структура — вернуть как есть
|
||
if isinstance(x, (dict, list, bool, int, float)) or x is None:
|
||
return x
|
||
s = str(x)
|
||
# Попробуем развернуть макросы/шаблон, если есть контекст
|
||
try:
|
||
s2 = render_template_simple(s, context or {}, out_map or {})
|
||
except Exception:
|
||
s2 = s
|
||
# Попробуем распарсить JSON
|
||
try:
|
||
return json.loads(s2)
|
||
except Exception:
|
||
return s2
|
||
|
||
def _fn_from_json(x: Any) -> Any:
|
||
val = _resolve_input(x)
|
||
if isinstance(val, str):
|
||
try:
|
||
return json.loads(val)
|
||
except Exception:
|
||
return val
|
||
return val
|
||
|
||
def _fn_jp(value: Any, path: Any, join_sep: Any = "\n") -> Any:
|
||
src = _resolve_input(value)
|
||
p = str(path or "")
|
||
try:
|
||
vals = _json_path_extract(src, p)
|
||
return vals
|
||
except Exception:
|
||
return None
|
||
|
||
def _fn_jp_text(value: Any, path: Any, join_sep: Any = "\n") -> str:
|
||
vals = _fn_jp(value, path, join_sep)
|
||
try:
|
||
return _stringify_join(vals, str(join_sep or "\n"))
|
||
except Exception:
|
||
# на крайний случай — строковое представление
|
||
try:
|
||
return str(vals)
|
||
except Exception:
|
||
return ""
|
||
|
||
# Новые функции для работы с изображениями
|
||
|
||
def _fn_file_b64(path_like: Any) -> str:
|
||
"""Прочитать файл и вернуть base64-строку (без префикса)."""
|
||
p_raw = _resolve_input(path_like)
|
||
p = Path(str(p_raw)).expanduser()
|
||
data = p.read_bytes()
|
||
return base64.b64encode(data).decode("ascii")
|
||
|
||
def _fn_data_url(b64: Any, mime: Any) -> str:
|
||
"""Собрать data URL: data:<mime>;base64,<b64>"""
|
||
b64s = str(_resolve_input(b64) if not isinstance(b64, str) else b64)
|
||
m = str(_resolve_input(mime) if not isinstance(mime, str) else mime).strip() or "application/octet-stream"
|
||
return f"data:{m};base64,{b64s}"
|
||
|
||
def _fn_file_data_url(path_like: Any, mime_opt: Any = None) -> str:
|
||
"""Скомбинировать чтение файла и сборку data URL. mime можно не указывать — попытка угадать по расширению."""
|
||
p_raw = _resolve_input(path_like)
|
||
p = Path(str(p_raw)).expanduser()
|
||
try:
|
||
data = p.read_bytes()
|
||
except Exception as e:
|
||
print(f"DEBUG: file_data_url error reading file {p_raw}: {e}")
|
||
raise
|
||
size = len(data)
|
||
print(f"DEBUG: file_data_url loaded file {p!s} size={size} bytes")
|
||
b64s = base64.b64encode(data).decode("ascii")
|
||
mime = None
|
||
if mime_opt is not None:
|
||
mime = str(_resolve_input(mime_opt))
|
||
if not mime:
|
||
mt, _ = mimetypes.guess_type(str(p))
|
||
mime = mt or "application/octet-stream"
|
||
return f"data:{mime};base64,{b64s}"
|
||
|
||
# 0) Попытаться распознать чистый JSON‑литерал (включая true/false/null, объекты/массивы/числа/строки).
|
||
try:
|
||
s = str(expr).strip()
|
||
return json.loads(s)
|
||
except Exception:
|
||
pass
|
||
|
||
allowed_bin = {
|
||
ast.Add: op.add, ast.Sub: op.sub, ast.Mult: op.mul, ast.Div: op.truediv,
|
||
ast.FloorDiv: op.floordiv, ast.Mod: op.mod,
|
||
}
|
||
allowed_unary = {ast.UAdd: lambda x: +x, ast.USub: lambda x: -x}
|
||
allowed_cmp = {
|
||
ast.Eq: op.eq, ast.NotEq: op.ne, ast.Lt: op.lt, ast.LtE: op.le, ast.Gt: op.gt, ast.GtE: op.ge,
|
||
}
|
||
|
||
def eval_node(node: ast.AST) -> Any:
|
||
if isinstance(node, ast.Expression):
|
||
return eval_node(node.body)
|
||
if isinstance(node, ast.Constant):
|
||
return node.value
|
||
if isinstance(node, ast.Tuple):
|
||
return tuple(eval_node(e) for e in node.elts)
|
||
if isinstance(node, ast.List):
|
||
return [eval_node(e) for e in node.elts]
|
||
if isinstance(node, ast.Dict):
|
||
return {eval_node(k): eval_node(v) for k, v in zip(node.keys, node.values)}
|
||
if isinstance(node, ast.UnaryOp) and type(node.op) in allowed_unary:
|
||
return allowed_unary[type(node.op)](eval_node(node.operand))
|
||
if isinstance(node, ast.BinOp) and type(node.op) in allowed_bin:
|
||
return allowed_bin[type(node.op)](eval_node(node.left), eval_node(node.right))
|
||
if isinstance(node, ast.BoolOp):
|
||
vals = [eval_node(v) for v in node.values]
|
||
if isinstance(node.op, ast.And):
|
||
res = True
|
||
for v in vals:
|
||
res = res and bool(v)
|
||
return res
|
||
if isinstance(node.op, ast.Or):
|
||
res = False
|
||
for v in vals:
|
||
res = res or bool(v)
|
||
return res
|
||
if isinstance(node, ast.Compare):
|
||
left = eval_node(node.left)
|
||
for opnode, comparator in zip(node.ops, node.comparators):
|
||
if type(opnode) not in allowed_cmp:
|
||
raise ExecutionError("Unsupported comparison operator")
|
||
right = eval_node(comparator)
|
||
if not allowed_cmp[type(opnode)](left, right):
|
||
return False
|
||
left = right
|
||
return True
|
||
# Разрешённые вызовы: rand(), randint(a,b), choice(list), jp(), jp_text(), from_json(), file_b64(), data_url(), file_data_url()
|
||
if isinstance(node, ast.Call):
|
||
# Никаких kwargs, *args
|
||
if node.keywords or isinstance(getattr(node, "starargs", None), ast.AST) or isinstance(getattr(node, "kwargs", None), ast.AST):
|
||
raise ExecutionError("Call with kwargs/starargs is not allowed")
|
||
fn = node.func
|
||
if not isinstance(fn, ast.Name):
|
||
raise ExecutionError("Only simple function calls are allowed")
|
||
name = fn.id
|
||
if name == "rand":
|
||
if len(node.args) != 0:
|
||
raise ExecutionError("rand() takes no arguments")
|
||
return random.random()
|
||
if name == "randint":
|
||
if len(node.args) != 2:
|
||
raise ExecutionError("randint(a,b) requires two arguments")
|
||
a = eval_node(node.args[0])
|
||
b = eval_node(node.args[1])
|
||
try:
|
||
return random.randint(int(a), int(b))
|
||
except Exception as exc: # noqa: BLE001
|
||
raise ExecutionError(f"randint invalid arguments: {exc}")
|
||
if name == "choice":
|
||
if len(node.args) != 1:
|
||
raise ExecutionError("choice(list) requires one argument")
|
||
seq = eval_node(node.args[0])
|
||
if not isinstance(seq, (list, tuple)):
|
||
raise ExecutionError("choice() expects list or tuple")
|
||
if not seq:
|
||
raise ExecutionError("choice() on empty sequence")
|
||
return random.choice(seq)
|
||
if name == "jp":
|
||
if len(node.args) < 2 or len(node.args) > 3:
|
||
raise ExecutionError("jp(value, path, [join_sep]) requires 2 or 3 args")
|
||
v = eval_node(node.args[0])
|
||
p = eval_node(node.args[1])
|
||
return _fn_jp(v, p, eval_node(node.args[2]) if len(node.args) == 3 else "\n")
|
||
if name == "jp_text":
|
||
if len(node.args) < 2 or len(node.args) > 3:
|
||
raise ExecutionError("jp_text(value, path, [join_sep]) requires 2 or 3 args")
|
||
v = eval_node(node.args[0])
|
||
p = eval_node(node.args[1])
|
||
return _fn_jp_text(v, p, eval_node(node.args[2]) if len(node.args) == 3 else "\n")
|
||
if name == "from_json":
|
||
if len(node.args) != 1:
|
||
raise ExecutionError("from_json(x) requires one argument")
|
||
v = eval_node(node.args[0])
|
||
return _fn_from_json(v)
|
||
if name == "file_b64":
|
||
if len(node.args) != 1:
|
||
raise ExecutionError("file_b64(path) requires one argument")
|
||
return _fn_file_b64(eval_node(node.args[0]))
|
||
if name == "data_url":
|
||
if len(node.args) != 2:
|
||
raise ExecutionError("data_url(b64, mime) requires two arguments")
|
||
b64v = eval_node(node.args[0])
|
||
mimev = eval_node(node.args[1])
|
||
return _fn_data_url(b64v, mimev)
|
||
if name == "file_data_url":
|
||
if len(node.args) < 1 or len(node.args) > 2:
|
||
raise ExecutionError("file_data_url(path, [mime]) requires 1 or 2 args")
|
||
p = eval_node(node.args[0])
|
||
mv = eval_node(node.args[1]) if len(node.args) == 2 else None
|
||
return _fn_file_data_url(p, mv)
|
||
raise ExecutionError(f"Function {name} is not allowed")
|
||
# Запрещаем всё остальное (Name/Attribute/Subscript/IfExp/Comprehensions и пр.)
|
||
raise ExecutionError("Expression not allowed")
|
||
|
||
try:
|
||
tree = ast.parse(str(expr), mode="eval")
|
||
except Exception as exc:
|
||
raise ExecutionError(f"SetVars expr parse error: {exc}") from exc
|
||
return eval_node(tree)
|
||
|
||
async def run(self, inputs: Dict[str, Any], context: Dict[str, Any]) -> Dict[str, Any]: # noqa: D401
|
||
out_map = context.get("OUT") or {}
|
||
result: Dict[str, Any] = {}
|
||
import re as _re
|
||
for v in sorted(self._normalize(), key=lambda x: x.get("order", 0)):
|
||
name = v.get("name") or ""
|
||
if not _re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name or ""):
|
||
raise ExecutionError(f"SetVars invalid variable name: {name!r}")
|
||
mode = v.get("mode", "string")
|
||
raw_val = v.get("value", "")
|
||
if mode == "expr":
|
||
resolved = self._safe_eval_expr(str(raw_val), context, out_map)
|
||
else:
|
||
resolved = render_template_simple(str(raw_val or ""), context, out_map)
|
||
result[name] = resolved
|
||
# Событие «vars_set» в SSE‑лог (+ превью значений)
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
def _pv(val: Any) -> str:
|
||
try:
|
||
if isinstance(val, str):
|
||
s = val
|
||
else:
|
||
s = json.dumps(val, ensure_ascii=False)
|
||
except Exception:
|
||
try:
|
||
s = str(val)
|
||
except Exception:
|
||
s = "<unrepresentable>"
|
||
s = s if isinstance(s, str) else str(s)
|
||
return (s[:300] + "…") if len(s) > 300 else s
|
||
values_preview = {k: _pv(v) for k, v in (result or {}).items()}
|
||
await trace_fn({
|
||
"event": "vars_set",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"vars": list(result.keys()),
|
||
"count": len(result),
|
||
"values_preview": values_preview,
|
||
})
|
||
except Exception:
|
||
pass
|
||
return {"vars": result}
|
||
|
||
# --- v1: Нормализованный экстрактор текста для OUTx -----------------------------
|
||
def _json_path_extract(obj: Any, path: str) -> Any:
|
||
if not path:
|
||
return None
|
||
nodes = [obj]
|
||
for raw_seg in str(path).split("."):
|
||
seg = (raw_seg or "").strip()
|
||
if not seg:
|
||
continue
|
||
next_nodes = []
|
||
# Поддержка "*": разворачиваем dict.values() или элементы списка
|
||
if seg == "*":
|
||
for n in nodes:
|
||
if isinstance(n, dict):
|
||
next_nodes.extend(list(n.values()))
|
||
elif isinstance(n, list):
|
||
next_nodes.extend(list(n))
|
||
nodes = next_nodes
|
||
if not nodes:
|
||
break
|
||
continue
|
||
# Числовой индекс для списков
|
||
idx = None
|
||
try:
|
||
idx = int(seg)
|
||
except Exception:
|
||
idx = None
|
||
for n in nodes:
|
||
if idx is not None and isinstance(n, list):
|
||
if 0 <= idx < len(n):
|
||
next_nodes.append(n[idx])
|
||
elif idx is None and isinstance(n, dict):
|
||
if seg in n:
|
||
next_nodes.append(n[seg])
|
||
nodes = next_nodes
|
||
if not nodes:
|
||
break
|
||
if not nodes:
|
||
return None
|
||
return nodes if len(nodes) > 1 else nodes[0]
|
||
|
||
|
||
def _stringify_join(values: Any, join_sep: str = "\n") -> str:
|
||
def _flatten(x):
|
||
if isinstance(x, list):
|
||
for it in x:
|
||
yield from _flatten(it)
|
||
else:
|
||
yield x
|
||
arr = list(_flatten(values if isinstance(values, list) else [values]))
|
||
out: List[str] = []
|
||
for v in arr:
|
||
if isinstance(v, str):
|
||
if v:
|
||
out.append(v)
|
||
elif isinstance(v, (dict, list)):
|
||
try:
|
||
t = _deep_find_text(v) # type: ignore[name-defined]
|
||
if isinstance(t, str) and t:
|
||
out.append(t)
|
||
except Exception:
|
||
pass
|
||
else:
|
||
try:
|
||
sv = str(v)
|
||
if sv:
|
||
out.append(sv)
|
||
except Exception:
|
||
pass
|
||
if not out:
|
||
return ""
|
||
return (join_sep or "\n").join(out)
|
||
|
||
|
||
def _extract_text_for_out(data: Any, strategy: str = "auto", provider_hint: str = "", json_path: str = "", join_sep: str = "\n") -> str:
|
||
s = (strategy or "auto").lower().strip() or "auto"
|
||
prov = (provider_hint or "").lower().strip()
|
||
|
||
# Принудительные стратегии
|
||
if s == "deep":
|
||
try:
|
||
t = _deep_find_text(data) # type: ignore[name-defined]
|
||
return t if isinstance(t, str) else (str(t) if t is not None else "")
|
||
except Exception:
|
||
return ""
|
||
if s == "jsonpath":
|
||
try:
|
||
vals = _json_path_extract(data, json_path or "")
|
||
return _stringify_join(vals, join_sep)
|
||
except Exception:
|
||
return ""
|
||
if s in {"openai", "gemini", "claude"}:
|
||
try:
|
||
if s == "openai" and isinstance(data, dict):
|
||
msg = (data.get("choices") or [{}])[0].get("message") or {}
|
||
c = msg.get("content")
|
||
return c if isinstance(c, str) else (str(c) if c is not None else "")
|
||
if s == "gemini" and isinstance(data, dict):
|
||
cands = data.get("candidates") or []
|
||
texts: List[str] = []
|
||
for cand in cands:
|
||
try:
|
||
content = cand.get("content") or {}
|
||
parts = content.get("parts") or []
|
||
for p in parts:
|
||
if isinstance(p, dict):
|
||
t = p.get("text")
|
||
if isinstance(t, str) and t.strip():
|
||
texts.append(t.strip())
|
||
except Exception:
|
||
continue
|
||
if texts:
|
||
return (join_sep or "\n").join(texts)
|
||
return ""
|
||
if s == "claude" and isinstance(data, dict):
|
||
blocks = data.get("content") or []
|
||
texts = [b.get("text") for b in blocks if isinstance(b, dict) and isinstance(b.get("text"), str)]
|
||
if texts:
|
||
return "\n".join(texts)
|
||
return ""
|
||
except Exception:
|
||
return ""
|
||
return ""
|
||
|
||
# auto: если есть подсказка — пробуем сначала её ветку, потом общий
|
||
if prov in {"openai", "gemini", "claude"}:
|
||
forced = _extract_text_for_out(data, prov, prov, json_path, join_sep)
|
||
if forced:
|
||
return forced
|
||
|
||
# Универсальный best-effort: знает openai/gemini/claude
|
||
try:
|
||
t = _best_text_from_outputs({"result": data}) # type: ignore[name-defined]
|
||
if isinstance(t, str) and t:
|
||
return t
|
||
if t is not None:
|
||
t2 = str(t)
|
||
if t2:
|
||
return t2
|
||
except Exception:
|
||
pass
|
||
|
||
# Последний шанс — глубокий поиск
|
||
try:
|
||
t = _deep_find_text(data) # type: ignore[name-defined]
|
||
return t if isinstance(t, str) else (str(t) if t is not None else "")
|
||
except Exception:
|
||
return ""
|
||
|
||
class ProviderCallNode(Node):
|
||
type_name = "ProviderCall"
|
||
|
||
# --- Prompt Manager helpers -------------------------------------------------
|
||
def _get_blocks(self) -> List[Dict[str, Any]]:
|
||
"""Return normalized list of prompt blocks from config."""
|
||
raw = self.config.get("blocks") or self.config.get("prompt_blocks") or []
|
||
if not isinstance(raw, list):
|
||
return []
|
||
norm: List[Dict[str, Any]] = []
|
||
for i, b in enumerate(raw):
|
||
if not isinstance(b, dict):
|
||
continue
|
||
role = str(b.get("role", "user")).lower().strip()
|
||
if role not in {"system", "user", "assistant", "tool"}:
|
||
role = "user"
|
||
# order fallback: keep original index if not provided/correct
|
||
try:
|
||
order = int(b.get("order")) if b.get("order") is not None else i
|
||
except Exception: # noqa: BLE001
|
||
order = i
|
||
norm.append(
|
||
{
|
||
"id": b.get("id") or f"b{i}",
|
||
"name": b.get("name") or f"Block {i+1}",
|
||
"role": role,
|
||
"prompt": b.get("prompt") or "",
|
||
"enabled": bool(b.get("enabled", True)),
|
||
"order": order,
|
||
}
|
||
)
|
||
return norm
|
||
|
||
def _render_blocks_to_unified(self, context: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||
"""
|
||
Filter+sort+render blocks to unified messages:
|
||
[{role, content, name?}]
|
||
|
||
Расширение: поддержка изображений в Markdown вида .
|
||
Если в тексте встречаются такие конструкции — content превращается в список частей:
|
||
- {"type":"text","text":"..."}
|
||
- {"type":"image_url","url":"..."}
|
||
"""
|
||
import re as _re
|
||
out_map = context.get("OUT") or {}
|
||
blocks = [b for b in self._get_blocks() if b.get("enabled", True)]
|
||
blocks.sort(key=lambda x: x.get("order", 0))
|
||
|
||
def _parse_markdown_parts(s: str) -> List[Dict[str, Any]]:
|
||
print(f"DEBUG: _parse_markdown_parts input={s!r}")
|
||
pat = _re.compile(r"!\[[^\]]*\]\(([^)]+)\)")
|
||
parts: List[Dict[str, Any]] = []
|
||
pos = 0
|
||
for m in pat.finditer(s):
|
||
print(f"DEBUG: _parse_markdown_parts match at span={m.span()} url={m.group(1)!r}")
|
||
if m.start() > pos:
|
||
txt = s[pos:m.start()]
|
||
if txt:
|
||
parts.append({"type": "text", "text": txt})
|
||
url = (m.group(1) or "").strip()
|
||
if url:
|
||
parts.append({"type": "image_url", "url": url})
|
||
pos = m.end()
|
||
if pos < len(s):
|
||
tail = s[pos:]
|
||
if tail:
|
||
parts.append({"type": "text", "text": tail})
|
||
return parts
|
||
|
||
messages: List[Dict[str, Any]] = []
|
||
for b in blocks:
|
||
content_raw = render_template_simple(str(b.get("prompt") or ""), context, out_map)
|
||
parts = _parse_markdown_parts(content_raw or "")
|
||
# Если нет картинок — оставляем строкой для обратной совместимости
|
||
content_final: Any = parts if any(p.get("type") == "image_url" for p in parts) else content_raw
|
||
msg = {"role": b["role"], "content": content_final}
|
||
messages.append(msg)
|
||
return messages
|
||
|
||
def _messages_to_payload(self, provider: str, messages: List[Dict[str, Any]], context: Dict[str, Any]) -> Dict[str, Any]:
|
||
"""Convert unified messages (with optional text+image parts) to provider-specific request payload."""
|
||
import re as _re
|
||
params = context.get("params") or {}
|
||
model = context.get("model") or ""
|
||
|
||
def _is_data_url(u: str) -> bool:
|
||
return isinstance(u, str) and u.strip().lower().startswith("data:")
|
||
|
||
def _split_data_url(u: str) -> tuple[str, str]:
|
||
"""
|
||
Возвращает (mime, b64) для data URL.
|
||
Поддерживаем форму: data:<mime>;base64,<b64>
|
||
"""
|
||
try:
|
||
header, b64 = u.split(",", 1)
|
||
mime = "application/octet-stream"
|
||
if header.startswith("data:"):
|
||
header2 = header[5:]
|
||
if ";base64" in header2:
|
||
mime = header2.split(";base64", 1)[0] or mime
|
||
elif ";" in header2:
|
||
mime = header2.split(";", 1)[0] or mime
|
||
elif header2:
|
||
mime = header2
|
||
return mime, b64
|
||
except Exception:
|
||
return "application/octet-stream", ""
|
||
|
||
provider = (provider or "").lower()
|
||
|
||
if provider == "openai":
|
||
def _map_openai_msg(m: Dict[str, Any]) -> Dict[str, Any]:
|
||
c = m.get("content")
|
||
if isinstance(c, list):
|
||
parts = []
|
||
for p in c:
|
||
if isinstance(p, dict) and p.get("type") == "text":
|
||
parts.append({"type": "text", "text": str(p.get("text") or "")})
|
||
elif isinstance(p, dict) and p.get("type") in {"image_url", "image"}:
|
||
url = str(p.get("url") or "")
|
||
parts.append({"type": "image_url", "image_url": {"url": url}})
|
||
return {"role": m.get("role", "user"), "content": parts}
|
||
# строка — как было
|
||
return {"role": m.get("role", "user"), "content": str(c or "")}
|
||
|
||
payload: Dict[str, Any] = {
|
||
"model": model,
|
||
"messages": [_map_openai_msg(m) for m in messages],
|
||
"temperature": params.get("temperature", 0.7),
|
||
}
|
||
if params.get("max_tokens") is not None:
|
||
payload["max_tokens"] = params.get("max_tokens")
|
||
if params.get("top_p") is not None:
|
||
payload["top_p"] = params.get("top_p")
|
||
if params.get("stop") is not None:
|
||
payload["stop"] = params.get("stop")
|
||
return payload
|
||
|
||
if provider == "gemini_image":
|
||
def _map_gemini_image_msg(m: Dict[str, Any]) -> Dict[str, Any]:
|
||
c = m.get("content")
|
||
parts: List[Dict[str, Any]] = []
|
||
if isinstance(c, list):
|
||
for p in c:
|
||
if isinstance(p, dict) and p.get("type") == "text":
|
||
parts.append({"text": str(p.get("text") or "")})
|
||
elif isinstance(p, dict) and p.get("type") in {"image_url", "image"}:
|
||
url = str(p.get("url") or "")
|
||
if _is_data_url(url):
|
||
mime, b64 = _split_data_url(url)
|
||
parts.append({"inline_data": {"mime_type": mime, "data": b64}})
|
||
else:
|
||
parts.append({"text": url})
|
||
else:
|
||
parts.append({"text": str(c or "")})
|
||
return {"parts": parts}
|
||
|
||
payload: Dict[str, Any] = {"model": model, "contents": [_map_gemini_image_msg(m) for m in messages]}
|
||
return payload
|
||
|
||
if provider == "gemini":
|
||
def _map_claude_msg(m: Dict[str, Any]) -> Dict[str, Any]:
|
||
role_raw = str(m.get("role") or "user")
|
||
role = role_raw if role_raw in {"user", "assistant"} else "user"
|
||
c = m.get("content")
|
||
blocks: List[Dict[str, Any]] = []
|
||
if isinstance(c, list):
|
||
for p in c:
|
||
if not isinstance(p, dict):
|
||
continue
|
||
if p.get("type") == "text":
|
||
blocks.append({"type": "text", "text": str(p.get("text") or "")})
|
||
elif p.get("type") in {"image_url", "image"}:
|
||
url = str(p.get("url") or "")
|
||
if _is_data_url(url):
|
||
mime, b64 = _split_data_url(url)
|
||
blocks.append({"type": "image", "source": {"type": "base64", "media_type": mime, "data": b64}})
|
||
else:
|
||
blocks.append({"type": "image", "source": {"type": "url", "url": url}})
|
||
else:
|
||
blocks.append({"type": "text", "text": str(c or "")})
|
||
return {"role": role, "content": blocks}
|
||
|
||
# system — текстовая склейка (без картинок)
|
||
sys_text = "\n\n".join(
|
||
[str(m.get("content") or "") if not isinstance(m.get("content"), list)
|
||
else "\n".join([str(p.get("text") or "") for p in m.get("content") if isinstance(p, dict) and p.get("type") == "text"])
|
||
for m in messages if m.get("role") == "system"]
|
||
).strip()
|
||
|
||
msgs = [_map_claude_msg(m) for m in messages if m.get("role") != "system"]
|
||
payload: Dict[str, Any] = {
|
||
"model": model,
|
||
"messages": msgs,
|
||
}
|
||
if sys_text:
|
||
payload["system"] = sys_text
|
||
if params.get("temperature") is not None:
|
||
payload["temperature"] = params.get("temperature")
|
||
if params.get("max_tokens") is not None:
|
||
payload["max_tokens"] = params.get("max_tokens")
|
||
if params.get("top_p") is not None:
|
||
payload["top_p"] = params.get("top_p")
|
||
if params.get("stop") is not None:
|
||
payload["stop"] = params.get("stop")
|
||
return payload
|
||
|
||
return {}
|
||
|
||
def _blocks_struct_for_template(self, provider: str, messages: List[Dict[str, Any]], context: Dict[str, Any]) -> Dict[str, Any]:
|
||
"""
|
||
Сформировать структуру для вставки в шаблон (template) из Prompt Blocks.
|
||
Возвращает provider-специфичные ключи, которые можно вставлять в JSON:
|
||
- openai: { "messages": [...] , "system_text": "..." }
|
||
- gemini: { "contents": [...], "systemInstruction": {...}, "system_text": "..." }
|
||
- claude: { "system_text": "...", "system": "...", "messages": [...] }
|
||
|
||
Расширение: поддержка текст+изображения, как и в _messages_to_payload.
|
||
"""
|
||
provider = (provider or "openai").lower()
|
||
msgs = messages or []
|
||
|
||
def _is_data_url(u: str) -> bool:
|
||
return isinstance(u, str) and u.strip().lower().startswith("data:")
|
||
|
||
def _split_data_url(u: str) -> tuple[str, str]:
|
||
try:
|
||
header, b64 = u.split(",", 1)
|
||
mime = "application/octet-stream"
|
||
if header.startswith("data:"):
|
||
header2 = header[5:]
|
||
if ";base64" in header2:
|
||
mime = header2.split(";base64", 1)[0] or mime
|
||
elif ";" in header2:
|
||
mime = header2.split(";", 1)[0] or mime
|
||
elif header2:
|
||
mime = header2
|
||
return mime, b64
|
||
except Exception:
|
||
return "application/octet-stream", ""
|
||
|
||
if provider == "openai":
|
||
def _map(m: Dict[str, Any]) -> Dict[str, Any]:
|
||
c = m.get("content")
|
||
if isinstance(c, list):
|
||
parts = []
|
||
for p in c:
|
||
if isinstance(p, dict) and p.get("type") == "text":
|
||
parts.append({"type": "text", "text": str(p.get("text") or "")})
|
||
elif isinstance(p, dict) and p.get("type") in {"image_url", "image"}:
|
||
url = str(p.get("url") or "")
|
||
parts.append({"type": "image_url", "image_url": {"url": url}})
|
||
return {"role": m.get("role", "user"), "content": parts}
|
||
return {"role": m.get("role", "user"), "content": str(c or "")}
|
||
sys_text = "\n\n".join(
|
||
[str(m.get("content") or "") if not isinstance(m.get("content"), list)
|
||
else "\n".join([str(p.get("text") or "") for p in m.get("content") if isinstance(p, dict) and p.get("type") == "text"])
|
||
for m in msgs if m.get("role") == "system"]
|
||
).strip()
|
||
return {
|
||
"messages": [_map(m) for m in msgs],
|
||
"system_text": sys_text,
|
||
}
|
||
|
||
if provider in {"gemini", "gemini_image"}:
|
||
def _text_from_msg(m: Dict[str, Any]) -> str:
|
||
c = m.get("content")
|
||
if isinstance(c, list):
|
||
texts = [str(p.get("text") or "") for p in c if isinstance(p, dict) and p.get("type") == "text"]
|
||
return "\n".join([t for t in texts if t])
|
||
return str(c or "")
|
||
sys_text = "\n\n".join([_text_from_msg(m) for m in msgs if m.get("role") == "system"]).strip()
|
||
|
||
contents = []
|
||
for m in msgs:
|
||
if m.get("role") == "system":
|
||
continue
|
||
role = "model" if m.get("role") == "assistant" else "user"
|
||
c = m.get("content")
|
||
parts: List[Dict[str, Any]] = []
|
||
if isinstance(c, list):
|
||
for p in c:
|
||
if not isinstance(p, dict):
|
||
continue
|
||
if p.get("type") == "text":
|
||
parts.append({"text": str(p.get("text") or "")})
|
||
elif p.get("type") in {"image_url", "image"}:
|
||
url = str(p.get("url") or "")
|
||
if _is_data_url(url):
|
||
mime, b64 = _split_data_url(url)
|
||
parts.append({"inline_data": {"mime_type": mime, "data": b64}})
|
||
else:
|
||
parts.append({"text": url})
|
||
else:
|
||
parts.append({"text": str(c or "")})
|
||
contents.append({"role": role, "parts": parts})
|
||
|
||
d: Dict[str, Any] = {
|
||
"contents": contents,
|
||
"system_text": sys_text,
|
||
}
|
||
if sys_text:
|
||
d["systemInstruction"] = {"parts": [{"text": sys_text}]}
|
||
return d
|
||
|
||
if provider == "claude":
|
||
# Системные сообщения как текст
|
||
sys_msgs = []
|
||
for m in msgs:
|
||
if m.get("role") == "system":
|
||
c = m.get("content")
|
||
if isinstance(c, list):
|
||
sys_msgs.append("\n".join([str(p.get("text") or "") for p in c if isinstance(p, dict) and p.get("type") == "text"]))
|
||
else:
|
||
sys_msgs.append(str(c or ""))
|
||
sys_text = "\n\n".join([s for s in sys_msgs if s]).strip()
|
||
|
||
out_msgs = []
|
||
for m in msgs:
|
||
if m.get("role") == "system":
|
||
continue
|
||
role = m.get("role")
|
||
role = role if role in {"user", "assistant"} else "user"
|
||
c = m.get("content")
|
||
blocks: List[Dict[str, Any]] = []
|
||
if isinstance(c, list):
|
||
for p in c:
|
||
if not isinstance(p, dict):
|
||
continue
|
||
if p.get("type") == "text":
|
||
blocks.append({"type": "text", "text": str(p.get("text") or "")})
|
||
elif p.get("type") in {"image_url", "image"}:
|
||
url = str(p.get("url") or "")
|
||
if _is_data_url(url):
|
||
mime, b64 = _split_data_url(url)
|
||
blocks.append({"type": "image", "source": {"type": "base64", "media_type": mime, "data": b64}})
|
||
else:
|
||
blocks.append({"type": "image", "source": {"type": "url", "url": url}})
|
||
else:
|
||
blocks.append({"type": "text", "text": str(c or "")})
|
||
out_msgs.append({"role": role, "content": blocks})
|
||
|
||
# Optional Claude mode: omit top-level "system" and prepend it as a user message
|
||
claude_no_system = False
|
||
try:
|
||
claude_no_system = bool((self.config or {}).get("claude_no_system", False))
|
||
except Exception:
|
||
claude_no_system = False
|
||
if claude_no_system:
|
||
if sys_text:
|
||
out_msgs = [{"role": "user", "content": [{"type": "text", "text": sys_text}]}] + out_msgs
|
||
return {
|
||
"messages": out_msgs,
|
||
"system_text": sys_text,
|
||
}
|
||
d = {
|
||
"system_text": sys_text,
|
||
"messages": out_msgs,
|
||
}
|
||
if sys_text:
|
||
# Prefer system as a plain string (proxy compatibility)
|
||
d["system"] = sys_text
|
||
return d
|
||
|
||
# По умолчанию ничего, но это валидный JSON
|
||
return {"messages": []}
|
||
|
||
async def run(self, inputs: Dict[str, Any], context: Dict[str, Any]) -> Dict[str, Any]:
|
||
# While-loop wrapper: if while_expr is set or ignore_errors enabled, route through helper
|
||
try:
|
||
cfg = self.config or {}
|
||
except Exception:
|
||
cfg = {}
|
||
try:
|
||
_while_expr = str(cfg.get("while_expr") or "").strip()
|
||
except Exception:
|
||
_while_expr = ""
|
||
try:
|
||
_ignore = bool(cfg.get("ignore_errors", False))
|
||
except Exception:
|
||
_ignore = False
|
||
if not (context.get("_in_while") or False) and (_while_expr or _ignore):
|
||
return await _providercall_run_with_while(self, inputs, context)
|
||
|
||
provider = (self.config.get("provider") or "openai").lower()
|
||
|
||
# Optional sleep before execution (UI-configured, milliseconds)
|
||
try:
|
||
sleep_ms = int(self.config.get("sleep_ms") or 0)
|
||
except Exception:
|
||
sleep_ms = 0
|
||
if sleep_ms > 0:
|
||
# Emit SSE event to highlight node as "sleeping" on UI
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
await trace_fn({
|
||
"event": "node_sleep",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"sleep_ms": int(sleep_ms),
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
# Non-blocking pause
|
||
try:
|
||
await asyncio.sleep(max(0.0, sleep_ms / 1000.0))
|
||
except Exception:
|
||
pass
|
||
|
||
# Support provider-specific configs stored in UI as provider_configs.{provider}
|
||
prov_cfg: Dict[str, Any] = {}
|
||
try:
|
||
cfgs = self.config.get("provider_configs") or {}
|
||
if isinstance(cfgs, dict):
|
||
prov_cfg = cfgs.get(provider) or {}
|
||
except Exception: # noqa: BLE001
|
||
prov_cfg = {}
|
||
|
||
base_url = prov_cfg.get("base_url") or self.config.get("base_url")
|
||
if not base_url:
|
||
raise ExecutionError(f"Node {self.node_id} ({self.type_name}) requires 'base_url' in config")
|
||
if not str(base_url).startswith(("http://", "https://")):
|
||
base_url = "http://" + str(base_url)
|
||
|
||
endpoint_tmpl: str = prov_cfg.get("endpoint") or self.config.get("endpoint") or ""
|
||
template: str = prov_cfg.get("template") or self.config.get("template") or "{}"
|
||
headers_json: str = prov_cfg.get("headers") or self.config.get("headers") or "{}"
|
||
|
||
# Default endpoints if not set
|
||
if not endpoint_tmpl:
|
||
_ad = None
|
||
try:
|
||
_ad = get_adapter(provider)
|
||
except Exception:
|
||
_ad = None
|
||
if _ad is not None:
|
||
try:
|
||
endpoint_tmpl = _ad.default_endpoint(str(context.get("model") or ""))
|
||
except Exception:
|
||
endpoint_tmpl = ""
|
||
if not endpoint_tmpl:
|
||
if provider == "openai":
|
||
endpoint_tmpl = "/v1/chat/completions"
|
||
elif provider == "gemini":
|
||
endpoint_tmpl = "/v1beta/models/{{ model }}:generateContent"
|
||
elif provider == "gemini_image":
|
||
# Специальный провайдер для генерации/превью изображений Gemini (generateContent)
|
||
endpoint_tmpl = "/v1beta/models/{{ model }}:generateContent"
|
||
elif provider == "claude":
|
||
endpoint_tmpl = "/v1/messages"
|
||
|
||
# Default template for gemini_image if none provided (inject [[PROMPT]])
|
||
try:
|
||
_tmpl_src = (prov_cfg.get("template") or self.config.get("template") or "").strip()
|
||
except Exception:
|
||
_tmpl_src = ""
|
||
if provider == "gemini_image" and (not _tmpl_src or _tmpl_src == "{}"):
|
||
# Минимальный валидный JSON с [[PROMPT]] для contents/systemInstruction
|
||
template = '{ "model": "{{ model }}", [[PROMPT]] }'
|
||
|
||
# Подготовим Prompt Blocks + pm-структуру для шаблона
|
||
unified_msgs = self._render_blocks_to_unified(context)
|
||
adapter = None
|
||
try:
|
||
adapter = get_adapter(provider)
|
||
except Exception:
|
||
adapter = None
|
||
if adapter:
|
||
blocks_struct = adapter.blocks_struct_for_template(unified_msgs, context, self.config or {})
|
||
else:
|
||
blocks_struct = self._blocks_struct_for_template(provider, unified_msgs, context)
|
||
pm_struct = dict(blocks_struct)
|
||
|
||
# Расширяем контекст для рендера шаблонов
|
||
render_ctx = dict(context)
|
||
render_ctx["pm"] = pm_struct
|
||
# Прокинем конфиг ноды в контекст для адаптеров (например, claude_no_system)
|
||
try:
|
||
render_ctx["_node_config"] = dict(self.config or {})
|
||
except Exception:
|
||
render_ctx["_node_config"] = {}
|
||
# Node-local: track VAR paths overridden by prompt_preprocess (no extra syntax)
|
||
pre_var_paths = set()
|
||
|
||
# prompt_preprocess (pre-merge DSL): парсим строки до prompt_combine и готовим «пред‑сегменты»
|
||
# Синтаксис строки:
|
||
# SEGMENT [delKeyContains "needle"] [delpos=prepend|append|N|-1] [pruneEmpty] [case=ci|cs]
|
||
# По умолчанию: case=ci, pruneEmpty=false, без delpos → append
|
||
# SEGMENT поддерживает [[...]] и {{ ... }}
|
||
pre_segments_raw: List[Dict[str, Any]] = []
|
||
try:
|
||
pre_raw = str((self.config or {}).get("prompt_preprocess") or "").strip()
|
||
except Exception:
|
||
pre_raw = ""
|
||
if pre_raw:
|
||
lines = [ln.strip() for ln in pre_raw.splitlines() if str(ln or "").strip()]
|
||
import re as _repp
|
||
# Collect local overrides for plain [[VAR:path]] segments after filtering
|
||
pre_var_overrides: Dict[str, Any] = {}
|
||
|
||
def _pp_try_json(s: str) -> Any:
|
||
try:
|
||
obj = json.loads(s)
|
||
except Exception:
|
||
try:
|
||
obj = json.loads(s, strict=False) # type: ignore[call-arg]
|
||
except Exception:
|
||
return None
|
||
for _ in range(2):
|
||
if isinstance(obj, str):
|
||
st = obj.strip()
|
||
if (st.startswith("{") and st.endswith("}")) or (st.startswith("[") and st.endswith("]")):
|
||
try:
|
||
obj = json.loads(st)
|
||
continue
|
||
except Exception:
|
||
break
|
||
break
|
||
return obj
|
||
|
||
def _norm(s: Any, ci: bool) -> str:
|
||
try:
|
||
ss = s if isinstance(s, str) else str(s)
|
||
except Exception:
|
||
ss = ""
|
||
return ss.lower() if ci else ss
|
||
|
||
def _delkeys_by_val_contains(x: Any, needles: List[str], ci: bool, prune_empty: bool, is_root: bool = False) -> tuple[Any, int]:
|
||
"""
|
||
Рекурсивно удаляет ключи словаря, если строковое представление их значения содержит needle.
|
||
- needles: список подстрок
|
||
- ci: регистронезависимый поиск (по умолчанию True)
|
||
- prune_empty: удалять пустые {} / [] из родителей (кроме корня)
|
||
Возвращает (новое_значение_или_None, удалённые_ключи_суммарно)
|
||
"""
|
||
removed = 0
|
||
if isinstance(x, dict):
|
||
out: Dict[str, Any] = {}
|
||
for k, v in x.items():
|
||
v2, rem2 = _delkeys_by_val_contains(v, needles, ci, prune_empty, False)
|
||
removed += rem2
|
||
sv = _stringify_for_template(v2)
|
||
cond = False
|
||
try:
|
||
nsv = _norm(sv, ci)
|
||
for nd in needles:
|
||
nds = _norm(nd, ci)
|
||
if nds and (nds in nsv):
|
||
cond = True
|
||
break
|
||
except Exception:
|
||
cond = False
|
||
if cond:
|
||
removed += 1
|
||
continue
|
||
out[k] = v2
|
||
if prune_empty and not is_root and len(out) == 0:
|
||
return None, removed # type: ignore[return-value]
|
||
return out, removed
|
||
if isinstance(x, list):
|
||
arr: List[Any] = []
|
||
for it in x:
|
||
it2, rem2 = _delkeys_by_val_contains(it, needles, ci, prune_empty, False)
|
||
removed += rem2
|
||
if it2 is None and prune_empty:
|
||
continue
|
||
arr.append(it2)
|
||
if prune_empty and not is_root and len(arr) == 0:
|
||
return None, removed # type: ignore[return-value]
|
||
return arr, removed
|
||
# scalar
|
||
return x, removed
|
||
|
||
pre_used = 0
|
||
pre_removed_total = 0
|
||
for ln in lines:
|
||
try:
|
||
# Опции
|
||
needles = [m.group(1) for m in _repp.finditer(r'(?is)\bdelKeyContains\s+"([^"]*)"', ln)]
|
||
mpos = _repp.search(r'(?is)\bdelpos\s*=\s*(prepend|append|-?\d+)\b', ln)
|
||
pos_spec = mpos.group(1).strip().lower() if mpos else None
|
||
mcase = _repp.search(r'(?is)\bcase\s*=\s*(ci|cs)\b', ln)
|
||
ci = True if not mcase else (mcase.group(1).strip().lower() == "ci")
|
||
prune = bool(_repp.search(r'(?is)\bpruneEmpty\b', ln))
|
||
|
||
# Очищаем директивы из текста строки → остаётся сам SEGMENT
|
||
s2 = _repp.sub(r'(?is)\bdelKeyContains\s+"[^"]*"', "", ln)
|
||
s2 = _repp.sub(r'(?is)\bdelpos\s*=\s*(prepend|append|-?\d+)\b', "", s2)
|
||
s2 = _repp.sub(r'(?is)\bcase\s*=\s*(ci|cs)\b', "", s2)
|
||
s2 = _repp.sub(r'(?is)\bpruneEmpty\b', "", s2)
|
||
seg = s2.strip()
|
||
if not seg:
|
||
continue
|
||
# Try to detect plain [[VAR:path]] to support node-local override without extra syntax
|
||
var_path = None
|
||
try:
|
||
mvar = _VAR_MACRO_RE.fullmatch(seg)
|
||
if mvar:
|
||
var_path = (mvar.group(1) or "").strip()
|
||
except Exception:
|
||
var_path = None
|
||
|
||
# Макросы и попытка распарсить JSON
|
||
resolved = render_template_simple(seg, render_ctx, render_ctx.get("OUT") or {})
|
||
obj = _pp_try_json(resolved)
|
||
base = obj if obj is not None else resolved
|
||
|
||
# Удаление ключей по contains, если задано
|
||
if needles:
|
||
try:
|
||
base2, remcnt = _delkeys_by_val_contains(base, needles, ci, prune, True)
|
||
except Exception:
|
||
base2, remcnt = base, 0
|
||
else:
|
||
base2, remcnt = base, 0
|
||
|
||
# If the segment was a pure [[VAR:path]] and we had filters,
|
||
# locally override this VAR for the rest of the node (so prompt_combine sees the filtered value)
|
||
try:
|
||
if var_path and needles:
|
||
pre_var_overrides[var_path] = base2
|
||
except Exception:
|
||
pass
|
||
|
||
pre_segments_raw.append({"obj": base2, "pos": pos_spec})
|
||
pre_used += 1
|
||
pre_removed_total += int(remcnt or 0)
|
||
except Exception:
|
||
continue
|
||
|
||
# Apply local VAR overrides onto render_ctx (node-local, no extra syntax)
|
||
try:
|
||
import copy as _copy
|
||
|
||
def _safe_deepcopy(x: Any) -> Any:
|
||
try:
|
||
return _copy.deepcopy(x)
|
||
except Exception:
|
||
# Fallback deep copy for dict/list; otherwise return as-is
|
||
try:
|
||
if isinstance(x, dict):
|
||
return {k: _safe_deepcopy(v) for k, v in x.items()}
|
||
if isinstance(x, list):
|
||
return [_safe_deepcopy(i) for i in x]
|
||
except Exception:
|
||
pass
|
||
try:
|
||
return json.loads(json.dumps(x))
|
||
except Exception:
|
||
return x
|
||
|
||
def _set_by_path(obj: Any, path: str, value: Any) -> None:
|
||
cur = obj
|
||
parts = [p.strip() for p in str(path).split(".") if p.strip()]
|
||
for i, part in enumerate(parts):
|
||
# list index?
|
||
idx = None
|
||
try:
|
||
idx = int(part)
|
||
except Exception:
|
||
idx = None
|
||
last = (i == len(parts) - 1)
|
||
if idx is not None:
|
||
if not isinstance(cur, list) or idx < 0 or idx >= len(cur):
|
||
return
|
||
if last:
|
||
cur[idx] = value
|
||
return
|
||
cur = cur[idx]
|
||
continue
|
||
# dict key
|
||
if not isinstance(cur, dict):
|
||
return
|
||
if last:
|
||
cur[part] = value
|
||
return
|
||
if part not in cur or not isinstance(cur[part], (dict, list)):
|
||
cur[part] = {}
|
||
cur = cur[part]
|
||
|
||
# Deep-copy only the top-level roots we are going to mutate (e.g., 'incoming' for 'incoming.*')
|
||
roots_to_copy: set = set()
|
||
for _p in (pre_var_overrides or {}).keys():
|
||
if "." in str(_p):
|
||
roots_to_copy.add(str(_p).split(".", 1)[0].strip())
|
||
for _root in roots_to_copy:
|
||
try:
|
||
if _root in render_ctx:
|
||
render_ctx[_root] = _safe_deepcopy(render_ctx[_root])
|
||
except Exception:
|
||
pass
|
||
|
||
for _p, _v in (pre_var_overrides or {}).items():
|
||
_set_by_path(render_ctx, _p, _v)
|
||
pre_var_paths = set(pre_var_overrides.keys())
|
||
except Exception:
|
||
pre_var_paths = set()
|
||
# SSE: prompt_preprocess summary
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
await trace_fn({
|
||
"event": "prompt_preprocess",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"provider": provider,
|
||
"lines": len(lines),
|
||
"used": pre_used,
|
||
"removed_keys": pre_removed_total,
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
else:
|
||
pre_segments_raw = []
|
||
|
||
# prompt_combine (DSL "&"): комбинируем сегменты в заданном порядке.
|
||
# Расширения:
|
||
# - [[PROMPT]] как специальный сегмент (позиционно вставляет pm_struct)
|
||
# - Директивы @pos=prepend|append|<idx> (управление позицией в массиве)
|
||
# - Фильтрация пустых сообщений/частей
|
||
try:
|
||
cfg = self.config or {}
|
||
combine_raw = str(cfg.get("prompt_combine") or "").strip()
|
||
except Exception:
|
||
combine_raw = ""
|
||
# Быстрая ветка: если есть адаптер — выполняем merge сегментов через него и коротко-замыкаем легаси-блок ниже
|
||
raw_segs_for_adapter = [s.strip() for s in combine_raw.split("&") if str(s or "").strip()]
|
||
if adapter and (raw_segs_for_adapter or (pre_segments_raw and len(pre_segments_raw) > 0)):
|
||
try:
|
||
pm_struct = adapter.combine_segments(
|
||
blocks_struct=blocks_struct,
|
||
pre_segments_raw=pre_segments_raw,
|
||
raw_segs=raw_segs_for_adapter,
|
||
render_ctx=render_ctx,
|
||
pre_var_paths=pre_var_paths,
|
||
render_template_simple_fn=render_template_simple,
|
||
var_macro_fullmatch_re=_VAR_MACRO_RE,
|
||
detect_vendor_fn=detect_vendor,
|
||
)
|
||
# обновим pm в контексте
|
||
try:
|
||
render_ctx["pm"] = pm_struct
|
||
except Exception:
|
||
pass
|
||
except Exception as _e:
|
||
try:
|
||
print(f"TRACE adapter_combine_error: node={self.node_id} provider={provider} err={_e}")
|
||
except Exception:
|
||
pass
|
||
else:
|
||
# коротко-замкнём легаси-блок prompt_combine ниже
|
||
pre_segments_raw = []
|
||
combine_raw = ""
|
||
if combine_raw or (pre_segments_raw and len(pre_segments_raw) > 0):
|
||
raw_segs = [s.strip() for s in combine_raw.split("&") if str(s or "").strip()]
|
||
|
||
def _try_json(s: str) -> Any:
|
||
"""
|
||
Try to parse JSON from a string.
|
||
Unwrap double-encoded JSON strings up to two passes and use a permissive fallback.
|
||
Returns parsed object (dict/list/primitive) or None on failure.
|
||
"""
|
||
try:
|
||
obj = json.loads(s)
|
||
except Exception:
|
||
# permissive fallback for odd control characters
|
||
try:
|
||
obj = json.loads(s, strict=False) # type: ignore[call-arg]
|
||
except Exception:
|
||
return None
|
||
# If decoded into a string that itself looks like JSON — try to parse again up to 2 times.
|
||
for _ in range(2):
|
||
if isinstance(obj, str):
|
||
st = obj.strip()
|
||
if (st.startswith("{") and st.endswith("}")) or (st.startswith("[") and st.endswith("]")):
|
||
try:
|
||
obj = json.loads(st)
|
||
continue
|
||
except Exception:
|
||
break
|
||
break
|
||
return obj
|
||
|
||
# Нормализация в формат провайдера
|
||
def _as_openai_messages(x: Any) -> List[Dict[str, Any]]:
|
||
msgs: List[Dict[str, Any]] = []
|
||
try:
|
||
# Dict inputs
|
||
if isinstance(x, dict):
|
||
if isinstance(x.get("messages"), list):
|
||
# Already OpenAI-style
|
||
return list(x.get("messages") or [])
|
||
if isinstance(x.get("contents"), list):
|
||
# Gemini -> OpenAI (text-only join)
|
||
for c in (x.get("contents") or []):
|
||
if not isinstance(c, dict):
|
||
continue
|
||
role_raw = str(c.get("role") or "user")
|
||
role = "assistant" if role_raw == "model" else ("user" if role_raw not in {"user","assistant"} else role_raw)
|
||
parts = c.get("parts") or []
|
||
text = "\n".join([str(p.get("text")) for p in parts if isinstance(p, dict) and isinstance(p.get("text"), str)]).strip()
|
||
msgs.append({"role": role, "content": text})
|
||
return msgs
|
||
|
||
# List inputs
|
||
if isinstance(x, list):
|
||
# Gemini contents list -> OpenAI messages
|
||
if all(isinstance(c, dict) and "parts" in c for c in x):
|
||
for c in x:
|
||
role_raw = str(c.get("role") or "user")
|
||
role = "assistant" if role_raw == "model" else ("user" if role_raw not in {"user","assistant"} else role_raw)
|
||
parts = c.get("parts") or []
|
||
text = "\n".join([str(p.get("text")) for p in parts if isinstance(p, dict) and isinstance(p.get("text"), str)]).strip()
|
||
msgs.append({"role": role, "content": text})
|
||
return msgs
|
||
# OpenAI messages list already — normalize parts if needed
|
||
if all(isinstance(m, dict) and "content" in m for m in x):
|
||
out: List[Dict[str, Any]] = []
|
||
for m in x:
|
||
role = m.get("role", "user")
|
||
cont = m.get("content")
|
||
if isinstance(cont, str):
|
||
out.append({"role": role, "content": cont})
|
||
elif isinstance(cont, list):
|
||
parts2: List[Dict[str, Any]] = []
|
||
for p in cont:
|
||
if not isinstance(p, dict):
|
||
continue
|
||
if p.get("type") == "text":
|
||
parts2.append({"type": "text", "text": str(p.get("text") or "")})
|
||
elif p.get("type") in {"image_url", "image"}:
|
||
# try OpenAI image_url shape
|
||
url = ""
|
||
if isinstance(p.get("image_url"), dict):
|
||
url = str((p.get("image_url") or {}).get("url") or "")
|
||
elif "url" in p:
|
||
url = str(p.get("url") or "")
|
||
if url:
|
||
parts2.append({"type": "image_url", "image_url": {"url": url}})
|
||
out.append({"role": role, "content": parts2 if parts2 else ""})
|
||
return out
|
||
# Fallback: dump JSON as a single user message
|
||
return [{"role":"user","content": json.dumps(x, ensure_ascii=False)}]
|
||
|
||
# Primitive inputs or embedded JSON string
|
||
if isinstance(x, str):
|
||
try_obj = _try_json(x)
|
||
if try_obj is not None:
|
||
return _as_openai_messages(try_obj)
|
||
return [{"role":"user","content": x}]
|
||
return [{"role":"user","content": json.dumps(x, ensure_ascii=False)}]
|
||
except Exception:
|
||
return [{"role":"user","content": str(x)}]
|
||
|
||
def _as_gemini_contents(x: Any) -> List[Dict[str, Any]]:
|
||
cnts: List[Dict[str, Any]] = []
|
||
try:
|
||
if isinstance(x, dict):
|
||
if isinstance(x.get("contents"), list):
|
||
return list(x.get("contents") or [])
|
||
if isinstance(x.get("messages"), list):
|
||
# OpenAI → Gemini
|
||
for m in (x.get("messages") or []):
|
||
if not isinstance(m, dict):
|
||
continue
|
||
role_raw = str(m.get("role") or "user")
|
||
role = "model" if role_raw == "assistant" else "user"
|
||
cont = m.get("content")
|
||
parts: List[Dict[str, Any]] = []
|
||
if isinstance(cont, str):
|
||
parts = [{"text": cont}]
|
||
elif isinstance(cont, list):
|
||
for p in cont:
|
||
if not isinstance(p, dict):
|
||
continue
|
||
if p.get("type") == "text":
|
||
parts.append({"text": str(p.get("text") or "")})
|
||
elif p.get("type") in {"image_url", "image"}:
|
||
# Gemini doesn't accept external image URLs directly — keep as text fallback
|
||
url = ""
|
||
if isinstance(p.get("image_url"), dict):
|
||
url = str((p.get("image_url") or {}).get("url") or "")
|
||
elif "url" in p:
|
||
url = str(p.get("url") or "")
|
||
if url:
|
||
parts.append({"text": url})
|
||
else:
|
||
parts = [{"text": json.dumps(cont, ensure_ascii=False)}]
|
||
cnts.append({"role": role, "parts": parts})
|
||
return cnts
|
||
|
||
if isinstance(x, list):
|
||
# Gemini contents list already
|
||
if all(isinstance(c, dict) and "parts" in c for c in x):
|
||
return list(x)
|
||
# OpenAI messages list -> Gemini
|
||
if all(isinstance(m, dict) and "content" in m for m in x):
|
||
out: List[Dict[str, Any]] = []
|
||
for m in x:
|
||
role_raw = str(m.get("role") or "user")
|
||
role = "model" if role_raw == "assistant" else "user"
|
||
cont = m.get("content")
|
||
parts: List[Dict[str, Any]] = []
|
||
if isinstance(cont, str):
|
||
parts = [{"text": cont}]
|
||
elif isinstance(cont, list):
|
||
for p in cont:
|
||
if not isinstance(p, dict):
|
||
continue
|
||
if p.get("type") == "text":
|
||
parts.append({"text": str(p.get("text") or "")})
|
||
elif p.get("type") in {"image_url", "image"}:
|
||
url = ""
|
||
if isinstance(p.get("image_url"), dict):
|
||
url = str((p.get("image_url") or {}).get("url") or "")
|
||
elif "url" in p:
|
||
url = str(p.get("url") or "")
|
||
if url:
|
||
parts.append({"text": url})
|
||
else:
|
||
parts = [{"text": json.dumps(cont, ensure_ascii=False)}]
|
||
out.append({"role": role, "parts": parts})
|
||
return out
|
||
# Fallback
|
||
return [{"role":"user","parts":[{"text": json.dumps(x, ensure_ascii=False)}]}]
|
||
|
||
if isinstance(x, str):
|
||
try_obj = _try_json(x)
|
||
if try_obj is not None:
|
||
return _as_gemini_contents(try_obj)
|
||
return [{"role":"user","parts":[{"text": x}]}]
|
||
return [{"role":"user","parts":[{"text": json.dumps(x, ensure_ascii=False)}]}]
|
||
except Exception:
|
||
return [{"role":"user","parts":[{"text": str(x)}]}]
|
||
|
||
def _as_claude_messages(x: Any) -> List[Dict[str, Any]]:
|
||
msgs: List[Dict[str, Any]] = []
|
||
try:
|
||
if isinstance(x, dict):
|
||
# Dict with messages (OpenAI-like)
|
||
if isinstance(x.get("messages"), list):
|
||
x = x.get("messages") or []
|
||
# fallthrough to list mapping below
|
||
elif isinstance(x.get("contents"), list):
|
||
# Gemini -> Claude
|
||
for c in (x.get("contents") or []):
|
||
if not isinstance(c, dict):
|
||
continue
|
||
role_raw = str(c.get("role") or "user")
|
||
role = "assistant" if role_raw == "model" else ("user" if role_raw not in {"user","assistant"} else role_raw)
|
||
parts = c.get("parts") or []
|
||
text = "\n".join([str(p.get("text")) for p in parts if isinstance(p, dict) and isinstance(p.get("text"), str)]).strip()
|
||
msgs.append({"role": role, "content":[{"type":"text","text": text}]})
|
||
return msgs
|
||
|
||
if isinstance(x, list):
|
||
# Gemini contents list -> Claude messages
|
||
if all(isinstance(c, dict) and "parts" in c for c in x):
|
||
for c in x:
|
||
role_raw = str(c.get("role") or "user")
|
||
role = "assistant" if role_raw == "model" else ("user" if role_raw not in {"user","assistant"} else role_raw)
|
||
blocks: List[Dict[str, Any]] = []
|
||
for p in (c.get("parts") or []):
|
||
if isinstance(p, dict) and isinstance(p.get("text"), str):
|
||
txt = p.get("text").strip()
|
||
if txt:
|
||
blocks.append({"type":"text","text": txt})
|
||
msgs.append({"role": role, "content": blocks or [{"type":"text","text": ""}]})
|
||
return msgs
|
||
# OpenAI messages list -> Claude
|
||
if all(isinstance(m, dict) and "content" in m for m in x):
|
||
out: List[Dict[str, Any]] = []
|
||
for m in x:
|
||
role = m.get("role", "user")
|
||
cont = m.get("content")
|
||
blocks: List[Dict[str, Any]] = []
|
||
if isinstance(cont, str):
|
||
blocks.append({"type":"text","text": cont})
|
||
elif isinstance(cont, list):
|
||
for p in cont:
|
||
if not isinstance(p, dict):
|
||
continue
|
||
if p.get("type") == "text":
|
||
blocks.append({"type":"text","text": str(p.get("text") or "")})
|
||
elif p.get("type") in {"image_url", "image"}:
|
||
url = ""
|
||
if isinstance(p.get("image_url"), dict):
|
||
url = str((p.get("image_url") or {}).get("url") or "")
|
||
elif "url" in p:
|
||
url = str(p.get("url") or "")
|
||
if url:
|
||
blocks.append({"type":"image","source":{"type":"url","url": url}})
|
||
else:
|
||
blocks.append({"type":"text","text": json.dumps(cont, ensure_ascii=False)})
|
||
out.append({"role": role if role in {"user","assistant"} else "user", "content": blocks})
|
||
return out
|
||
# Fallback
|
||
return [{"role":"user","content":[{"type":"text","text": json.dumps(x, ensure_ascii=False)}]}]
|
||
|
||
if isinstance(x, str):
|
||
try_obj = _try_json(x)
|
||
if try_obj is not None:
|
||
return _as_claude_messages(try_obj)
|
||
return [{"role":"user","content":[{"type":"text","text": x}]}]
|
||
return [{"role":"user","content":[{"type":"text","text": json.dumps(x, ensure_ascii=False)}]}]
|
||
except Exception:
|
||
return [{"role":"user","content":[{"type":"text","text": str(x)}]}]
|
||
|
||
# Helper: extract system text from mixed-provider fragments
|
||
def _extract_sys_text_from_obj(x: Any) -> Optional[str]:
|
||
"""
|
||
Extract textual 'system' from provider-like fragments:
|
||
- Claude: top-level 'system' (str or list of {type:'text', text:'...'})
|
||
- Gemini: 'systemInstruction' with parts[].text
|
||
- OpenAI: messages[*] with role == 'system' (string content or parts[].text)
|
||
Returns a string or None.
|
||
"""
|
||
try:
|
||
# 1) Dict objects
|
||
if isinstance(x, dict):
|
||
# Gemini systemInstruction
|
||
if "systemInstruction" in x:
|
||
si = x.get("systemInstruction")
|
||
def _parts_to_text(siobj: Any) -> str:
|
||
try:
|
||
parts = siobj.get("parts") or []
|
||
texts = [
|
||
str(p.get("text") or "")
|
||
for p in parts
|
||
if isinstance(p, dict) and isinstance(p.get("text"), str) and p.get("text").strip()
|
||
]
|
||
return "\n".join([t for t in texts if t]).strip()
|
||
except Exception:
|
||
return ""
|
||
if isinstance(si, dict):
|
||
t = _parts_to_text(si)
|
||
if t:
|
||
return t
|
||
if isinstance(si, list):
|
||
texts = []
|
||
for p in si:
|
||
if isinstance(p, dict) and isinstance(p.get("text"), str) and p.get("text").strip():
|
||
texts.append(p.get("text").strip())
|
||
t = "\n".join(texts).strip()
|
||
if t:
|
||
return t
|
||
if isinstance(si, str) and si.strip():
|
||
return si.strip()
|
||
# Claude system (string or blocks)
|
||
if "system" in x and not ("messages" in x and isinstance(x.get("messages"), list)):
|
||
sysv = x.get("system")
|
||
if isinstance(sysv, str) and sysv.strip():
|
||
return sysv.strip()
|
||
if isinstance(sysv, list):
|
||
texts = [
|
||
str(b.get("text") or "")
|
||
for b in sysv
|
||
if isinstance(b, dict)
|
||
and (b.get("type") == "text")
|
||
and isinstance(b.get("text"), str)
|
||
and b.get("text").strip()
|
||
]
|
||
t = "\n".join([t for t in texts if t]).strip()
|
||
if t:
|
||
return t
|
||
# OpenAI messages with role=system
|
||
if isinstance(x.get("messages"), list):
|
||
sys_msgs = []
|
||
for m in (x.get("messages") or []):
|
||
try:
|
||
if (str(m.get("role") or "").lower().strip() == "system"):
|
||
cont = m.get("content")
|
||
if isinstance(cont, str) and cont.strip():
|
||
sys_msgs.append(cont.strip())
|
||
elif isinstance(cont, list):
|
||
for p in cont:
|
||
if isinstance(p, dict) and p.get("type") == "text" and isinstance(p.get("text"), str) and p.get("text").strip():
|
||
sys_msgs.append(p.get("text").strip())
|
||
except Exception:
|
||
continue
|
||
if sys_msgs:
|
||
return "\n\n".join(sys_msgs).strip()
|
||
|
||
# 2) List objects
|
||
if isinstance(x, list):
|
||
# OpenAI messages list with role=system
|
||
if all(isinstance(m, dict) and "role" in m for m in x):
|
||
sys_msgs = []
|
||
for m in x:
|
||
try:
|
||
if (str(m.get("role") or "").lower().strip() == "system"):
|
||
cont = m.get("content")
|
||
if isinstance(cont, str) and cont.strip():
|
||
sys_msgs.append(cont.strip())
|
||
elif isinstance(cont, list):
|
||
for p in cont:
|
||
if isinstance(p, dict) and p.get("type") == "text" and isinstance(p.get("text"), str) and p.get("text").strip():
|
||
sys_msgs.append(p.get("text").strip())
|
||
except Exception:
|
||
continue
|
||
if sys_msgs:
|
||
return "\n\n".join(sys_msgs).strip()
|
||
# Gemini 'contents' list: try to read systemInstruction from incoming JSON snapshot
|
||
if all(isinstance(c, dict) and "parts" in c for c in x):
|
||
try:
|
||
inc = (render_ctx.get("incoming") or {}).get("json") or {}
|
||
si = inc.get("systemInstruction")
|
||
if si is not None:
|
||
return _extract_sys_text_from_obj({"systemInstruction": si})
|
||
except Exception:
|
||
pass
|
||
return None
|
||
except Exception:
|
||
return None
|
||
|
||
# Фильтр пустых сообщений/частей
|
||
def _filter_gemini(arr: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||
out: List[Dict[str, Any]] = []
|
||
for it in (arr or []):
|
||
if not isinstance(it, dict):
|
||
continue
|
||
parts = it.get("parts") or []
|
||
norm_parts = []
|
||
for p in parts:
|
||
if isinstance(p, dict):
|
||
t = p.get("text")
|
||
if isinstance(t, str) and t.strip():
|
||
norm_parts.append({"text": t})
|
||
elif "inline_data" in p or "inlineData" in p:
|
||
norm_parts.append(p) # изображения пропускаем как есть
|
||
if norm_parts:
|
||
out.append({"role": it.get("role","user"), "parts": norm_parts})
|
||
return out
|
||
|
||
def _filter_openai(arr: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||
out: List[Dict[str, Any]] = []
|
||
for m in (arr or []):
|
||
if not isinstance(m, dict):
|
||
continue
|
||
c = m.get("content")
|
||
if isinstance(c, str) and c.strip():
|
||
out.append({"role": m.get("role","user"), "content": c})
|
||
elif isinstance(c, list):
|
||
parts = []
|
||
for p in c:
|
||
if isinstance(p, dict) and p.get("type") == "text":
|
||
txt = str(p.get("text") or "")
|
||
if txt.strip():
|
||
parts.append({"type":"text","text": txt})
|
||
if parts:
|
||
out.append({"role": m.get("role","user"), "content": parts})
|
||
return out
|
||
|
||
def _filter_claude(arr: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||
out: List[Dict[str, Any]] = []
|
||
for m in (arr or []):
|
||
if not isinstance(m, dict):
|
||
continue
|
||
blocks = m.get("content")
|
||
if isinstance(blocks, list):
|
||
norm = []
|
||
for b in blocks:
|
||
if isinstance(b, dict) and b.get("type") == "text":
|
||
txt = str(b.get("text") or "")
|
||
if txt.strip():
|
||
norm.append({"type":"text","text": txt})
|
||
if norm:
|
||
out.append({"role": m.get("role","user"), "content": norm})
|
||
return out
|
||
|
||
# Построим массивы по сегментам в заданном порядке.
|
||
# Обработка [[PROMPT]] как спец.сегмента сохранив роль/части из pm_struct.
|
||
# Поддержка @pos=... только внутри текущего собираемого массива (messages/contents).
|
||
seg_items: List[Any] = []
|
||
# Снимок blocks как provider-структуры
|
||
blocks_struct = dict(pm_struct)
|
||
|
||
def _insert_items(base: List[Any], items: List[Any], pos_spec: Optional[str]) -> List[Any]:
|
||
if not items:
|
||
return base
|
||
if not pos_spec or pos_spec.lower() == "append":
|
||
base.extend(items)
|
||
return base
|
||
if pos_spec.lower() == "prepend":
|
||
return list(items) + base
|
||
# индекс
|
||
try:
|
||
idx = int(pos_spec)
|
||
if idx < 0:
|
||
idx = len(base) + idx
|
||
if idx < 0:
|
||
idx = 0
|
||
if idx > len(base): idx = len(base)
|
||
return base[:idx] + list(items) + base[idx:]
|
||
except Exception:
|
||
base.extend(items)
|
||
return base
|
||
|
||
# парсинг директивы @pos=
|
||
import re as _re2
|
||
def _split_pos_spec(s: str) -> tuple[str, Optional[str]]:
|
||
m = _re2.search(r"@pos\s*=\s*(prepend|append|-?\d+)\s*$", s, flags=_re2.IGNORECASE)
|
||
if not m:
|
||
return s.strip(), None
|
||
body = s[:m.start()].strip()
|
||
return body, m.group(1).strip().lower()
|
||
|
||
if provider in {"gemini", "gemini_image"}:
|
||
built: List[Dict[str, Any]] = []
|
||
sys_texts: List[str] = []
|
||
# Preprocess-inserted segments (prompt_preprocess)
|
||
for _pre in pre_segments_raw:
|
||
try:
|
||
_obj = _pre.get("obj")
|
||
items = _as_gemini_contents(_obj)
|
||
items = _filter_gemini(items)
|
||
built = _insert_items(built, items, _pre.get("pos"))
|
||
try:
|
||
sx = _extract_sys_text_from_obj(_obj)
|
||
if isinstance(sx, str) and sx.strip():
|
||
sys_texts.append(sx.strip())
|
||
except Exception:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
for raw_seg in raw_segs:
|
||
body_seg, pos_spec = _split_pos_spec(raw_seg)
|
||
if body_seg == "[[PROMPT]]":
|
||
items = _filter_gemini(list(blocks_struct.get("contents", []) or []))
|
||
built = _insert_items(built, items, pos_spec)
|
||
continue
|
||
m_pre = _VAR_MACRO_RE.fullmatch(body_seg)
|
||
if m_pre:
|
||
_p = (m_pre.group(1) or "").strip()
|
||
try:
|
||
if _p in pre_var_paths:
|
||
# Skip duplicate var segment - already inserted via prompt_preprocess (filtered)
|
||
continue
|
||
except Exception:
|
||
pass
|
||
resolved = render_template_simple(body_seg, render_ctx, render_ctx.get("OUT") or {})
|
||
obj = _try_json(resolved)
|
||
# provider guess + system extract for cross-provider combine
|
||
try:
|
||
pg = detect_vendor(obj if isinstance(obj, dict) else {})
|
||
print(f"DEBUG: prompt_combine seg provider_guess={pg} -> target=gemini pos={pos_spec}")
|
||
except Exception:
|
||
pass
|
||
items = _as_gemini_contents(obj if obj is not None else resolved)
|
||
items = _filter_gemini(items)
|
||
built = _insert_items(built, items, pos_spec)
|
||
try:
|
||
sx = _extract_sys_text_from_obj(obj) if obj is not None else None
|
||
if isinstance(sx, str) and sx.strip():
|
||
sys_texts.append(sx.strip())
|
||
except Exception:
|
||
pass
|
||
# Если ни один сегмент не был [[PROMPT]] и массив пуст — оставим исходные blocks
|
||
if not built:
|
||
built = _filter_gemini(list(blocks_struct.get("contents", []) or []))
|
||
# Merge systemInstruction: PROMPT blocks + gathered sys_texts
|
||
existing_si = blocks_struct.get("systemInstruction")
|
||
parts = []
|
||
if isinstance(existing_si, dict) and isinstance(existing_si.get("parts"), list):
|
||
parts = list(existing_si.get("parts") or [])
|
||
for s in sys_texts:
|
||
parts.append({"text": s})
|
||
new_si = {"parts": parts} if parts else existing_si
|
||
pm_struct = {"contents": built, "systemInstruction": new_si, "system_text": blocks_struct.get("system_text")}
|
||
elif provider in {"openai"}:
|
||
built2: List[Dict[str, Any]] = []
|
||
sys_texts: List[str] = []
|
||
# Preprocess-inserted segments (prompt_preprocess)
|
||
for _pre in pre_segments_raw:
|
||
try:
|
||
_obj = _pre.get("obj")
|
||
items = _as_openai_messages(_obj)
|
||
items = _filter_openai(items)
|
||
built2 = _insert_items(built2, items, _pre.get("pos"))
|
||
try:
|
||
sx = _extract_sys_text_from_obj(_obj)
|
||
if isinstance(sx, str) and sx.strip():
|
||
sys_texts.append(sx.strip())
|
||
except Exception:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
for raw_seg in raw_segs:
|
||
body_seg, pos_spec = _split_pos_spec(raw_seg)
|
||
if body_seg == "[[PROMPT]]":
|
||
items = _filter_openai(list(blocks_struct.get("messages", []) or []))
|
||
built2 = _insert_items(built2, items, pos_spec)
|
||
continue
|
||
m_pre = _VAR_MACRO_RE.fullmatch(body_seg)
|
||
if m_pre:
|
||
_p = (m_pre.group(1) or "").strip()
|
||
try:
|
||
if _p in pre_var_paths:
|
||
# Skip duplicate var segment - already inserted via prompt_preprocess (filtered)
|
||
continue
|
||
except Exception:
|
||
pass
|
||
resolved = render_template_simple(body_seg, render_ctx, render_ctx.get("OUT") or {})
|
||
obj = _try_json(resolved)
|
||
try:
|
||
pg = detect_vendor(obj if isinstance(obj, dict) else {})
|
||
print(f"DEBUG: prompt_combine seg provider_guess={pg} -> target=openai pos={pos_spec}")
|
||
except Exception:
|
||
pass
|
||
items = _as_openai_messages(obj if obj is not None else resolved)
|
||
items = _filter_openai(items)
|
||
built2 = _insert_items(built2, items, pos_spec)
|
||
# collect only top-level system/systemInstruction (not role=system in messages)
|
||
try:
|
||
sx = _extract_sys_text_from_obj(obj) if obj is not None else None
|
||
if isinstance(sx, str) and sx.strip():
|
||
sys_texts.append(sx.strip())
|
||
except Exception:
|
||
pass
|
||
if not built2:
|
||
built2 = _filter_openai(list(blocks_struct.get("messages", []) or []))
|
||
# Prepend system messages derived from sys_texts
|
||
if sys_texts:
|
||
sys_msgs = [{"role": "system", "content": s} for s in sys_texts if s]
|
||
if sys_msgs:
|
||
built2 = sys_msgs + built2
|
||
# keep system_text for UI/debug
|
||
st0 = blocks_struct.get("system_text") or ""
|
||
st = "\n\n".join([t for t in [st0] + sys_texts if isinstance(t, str) and t.strip()])
|
||
pm_struct = {"messages": built2, "system_text": st}
|
||
else: # claude
|
||
built3: List[Dict[str, Any]] = []
|
||
sys_texts: List[str] = []
|
||
# Preprocess-inserted segments (prompt_preprocess)
|
||
for _pre in pre_segments_raw:
|
||
try:
|
||
_obj = _pre.get("obj")
|
||
items = _as_claude_messages(_obj)
|
||
items = _filter_claude(items)
|
||
built3 = _insert_items(built3, items, _pre.get("pos"))
|
||
try:
|
||
sx = _extract_sys_text_from_obj(_obj)
|
||
if isinstance(sx, str) and sx.strip():
|
||
sys_texts.append(sx.strip())
|
||
except Exception:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
for raw_seg in raw_segs:
|
||
body_seg, pos_spec = _split_pos_spec(raw_seg)
|
||
if body_seg == "[[PROMPT]]":
|
||
items = _filter_claude(list(blocks_struct.get("messages", []) or []))
|
||
built3 = _insert_items(built3, items, pos_spec)
|
||
continue
|
||
m_pre = _VAR_MACRO_RE.fullmatch(body_seg)
|
||
if m_pre:
|
||
_p = (m_pre.group(1) or "").strip()
|
||
try:
|
||
if _p in pre_var_paths:
|
||
# Skip duplicate var segment - already inserted via prompt_preprocess (filtered)
|
||
continue
|
||
except Exception:
|
||
pass
|
||
resolved = render_template_simple(body_seg, render_ctx, render_ctx.get("OUT") or {})
|
||
obj = _try_json(resolved)
|
||
try:
|
||
pg = detect_vendor(obj if isinstance(obj, dict) else {})
|
||
print(f"DEBUG: prompt_combine seg provider_guess={pg} -> target=claude pos={pos_spec}")
|
||
except Exception:
|
||
pass
|
||
items = _as_claude_messages(obj if obj is not None else resolved)
|
||
items = _filter_claude(items)
|
||
built3 = _insert_items(built3, items, pos_spec)
|
||
try:
|
||
sx = _extract_sys_text_from_obj(obj) if obj is not None else None
|
||
if isinstance(sx, str) and sx.strip():
|
||
sys_texts.append(sx.strip())
|
||
except Exception:
|
||
pass
|
||
if not built3:
|
||
built3 = _filter_claude(list(blocks_struct.get("messages", []) or []))
|
||
# Merge system blocks from PROMPT blocks + gathered sys_texts
|
||
existing_sys = blocks_struct.get("system") or []
|
||
sys_blocks = []
|
||
if isinstance(existing_sys, list):
|
||
sys_blocks.extend(existing_sys)
|
||
st0 = blocks_struct.get("system_text") or ""
|
||
# Ensure PROMPT system_text from blocks is included as a Claude system block
|
||
if isinstance(st0, str) and st0.strip():
|
||
sys_blocks.append({"type": "text", "text": st0})
|
||
for s in sys_texts:
|
||
sys_blocks.append({"type": "text", "text": s})
|
||
st = "\n\n".join([t for t in [st0] + sys_texts if isinstance(t, str) and t.strip()])
|
||
|
||
# New: optional Claude mode to omit top-level "system"
|
||
claude_no_system = False
|
||
try:
|
||
claude_no_system = bool((self.config or {}).get("claude_no_system", False))
|
||
except Exception:
|
||
claude_no_system = False
|
||
|
||
if claude_no_system:
|
||
# Prepend system text as a user message instead of top-level system
|
||
if st:
|
||
built3 = [{"role": "user", "content": [{"type": "text", "text": st}]}] + built3
|
||
pm_struct = {"messages": built3, "system_text": st}
|
||
else:
|
||
# Prefer top-level system as plain string (proxy compatibility)
|
||
pm_struct = {"messages": built3, "system_text": st}
|
||
# Prefer array of system blocks when possible; fallback to single text block
|
||
if sys_blocks:
|
||
pm_struct["system"] = sys_blocks
|
||
elif st:
|
||
pm_struct["system"] = [{"type": "text", "text": st}]
|
||
|
||
# SSE метрика
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
after_cnt = (len(pm_struct.get("messages", [])) if provider in {"openai","claude"} else len(pm_struct.get("contents", [])))
|
||
await trace_fn({
|
||
"event": "prompt_merge",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"provider": provider,
|
||
"segments": len(raw_segs),
|
||
"before": (len(blocks_struct.get("messages", [])) if provider in {"openai","claude"} else len(blocks_struct.get("contents", []))),
|
||
"after": after_cnt,
|
||
"ts": int(time.time()*1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
|
||
# Единый JSON-фрагмент PROMPT для шаблонов: [[PROMPT]]
|
||
prompt_fragment = ""
|
||
try:
|
||
if adapter:
|
||
prompt_fragment = adapter.prompt_fragment(pm_struct, self.config or {})
|
||
else:
|
||
if provider == "openai":
|
||
prompt_fragment = '"messages": ' + json.dumps(pm_struct.get("messages", []), ensure_ascii=False)
|
||
elif provider in {"gemini", "gemini_image"}:
|
||
parts = []
|
||
contents = pm_struct.get("contents")
|
||
if contents is not None:
|
||
parts.append('"contents": ' + json.dumps(contents, ensure_ascii=False))
|
||
sysi = pm_struct.get("systemInstruction")
|
||
if sysi is not None:
|
||
parts.append('"systemInstruction": ' + json.dumps(sysi, ensure_ascii=False))
|
||
prompt_fragment = ", ".join(parts)
|
||
elif provider == "claude":
|
||
parts = []
|
||
# Учитываем флаг совместимости: при claude_no_system не добавляем top-level "system"
|
||
claude_no_system = False
|
||
try:
|
||
claude_no_system = bool((self.config or {}).get("claude_no_system", False))
|
||
except Exception:
|
||
claude_no_system = False
|
||
|
||
if not claude_no_system:
|
||
# Предпочитаем массив блоков system, если он есть; иначе строковый system_text
|
||
sys_val = pm_struct.get("system", None)
|
||
if sys_val is None:
|
||
sys_val = pm_struct.get("system_text")
|
||
if sys_val:
|
||
parts.append('"system": ' + json.dumps(sys_val, ensure_ascii=False))
|
||
|
||
msgs = pm_struct.get("messages")
|
||
if msgs is not None:
|
||
parts.append('"messages": ' + json.dumps(msgs, ensure_ascii=False))
|
||
prompt_fragment = ", ".join(parts)
|
||
except Exception: # noqa: BLE001
|
||
prompt_fragment = ""
|
||
render_ctx["PROMPT"] = prompt_fragment
|
||
|
||
# Render helper с поддержкой [[VAR]], [[OUT]] и {{ ... }}
|
||
def render(s: str) -> str:
|
||
return render_template_simple(s or "", render_ctx, render_ctx.get("OUT") or {})
|
||
|
||
# Рендер endpoint с макросами/шаблонами
|
||
endpoint = render(endpoint_tmpl)
|
||
|
||
# Формируем тело ТОЛЬКО из template/[[PROMPT]] (без сырого payload/входов).
|
||
# Больше НИКАКОГО фоллбэка на unified-построение: если шаблон невалиден — это ошибка ноды.
|
||
try:
|
||
rendered = render(template)
|
||
# DEBUG: печать отрендеренного шаблона с номерами строк для точной диагностики JSONDecodeError
|
||
try:
|
||
_lines = rendered.splitlines()
|
||
_preview = "\n".join(f"{i+1:03d}: {_lines[i]}" for i in range(min(len(_lines), 120)))
|
||
print(f"DEBUG: ProviderCallNode rendered_template node={self.node_id} provider={provider}\\n{_preview}")
|
||
except Exception:
|
||
try:
|
||
print(f"DEBUG: ProviderCallNode rendered_template(node={self.node_id}, provider={provider}) len={len(rendered)}")
|
||
except Exception:
|
||
pass
|
||
payload = json.loads(rendered)
|
||
except Exception as exc: # noqa: BLE001
|
||
raise ExecutionError(f"ProviderCall template invalid JSON: {exc}")
|
||
|
||
# Заголовки — полностью из редактируемого JSON с макросами
|
||
try:
|
||
headers_src = render(headers_json) if headers_json else "{}"
|
||
headers = json.loads(headers_src) if headers_src else {}
|
||
if not isinstance(headers, dict):
|
||
raise ValueError("headers must be a JSON object")
|
||
# Drop headers with empty values (proxy compatibility, e.g. "anthropic-beta": "")
|
||
try:
|
||
headers = {k: v for k, v in headers.items() if v is not None and (not isinstance(v, str) or v.strip() != "")}
|
||
except Exception:
|
||
pass
|
||
except Exception as exc: # noqa: BLE001
|
||
raise ExecutionError(f"ProviderCall headers invalid JSON: {exc}")
|
||
|
||
# Итоговый URL
|
||
if not base_url.startswith(("http://", "https://")):
|
||
base_url = "http://" + base_url
|
||
url = endpoint if endpoint.startswith("http") else urljoin(base_url.rstrip('/') + '/', endpoint.lstrip('/'))
|
||
|
||
# Debug logs to validate config selection and payload
|
||
# Brute request/response logging (request body sanitized like Burp)
|
||
try:
|
||
final_headers = {"Content-Type": "application/json", **headers}
|
||
print("===== ProviderCall REQUEST BEGIN =====")
|
||
print(f"node={self.node_id} type={self.type_name} provider={provider}")
|
||
print(f"URL: {url}")
|
||
try:
|
||
print("Headers:")
|
||
print(json.dumps(final_headers, ensure_ascii=False, indent=2))
|
||
except Exception:
|
||
print(f"Headers(raw): {final_headers}")
|
||
try:
|
||
san_payload = _sanitize_b64_for_log(payload, max_len=180)
|
||
print("Body JSON:")
|
||
print(json.dumps(san_payload, ensure_ascii=False, indent=2))
|
||
except Exception:
|
||
print(f"Body(raw): {payload}")
|
||
print("===== ProviderCall REQUEST END =====")
|
||
except Exception:
|
||
pass
|
||
# SSE: http_req (как в Burp)
|
||
req_id = f"{self.node_id}-{int(time.time()*1000)}"
|
||
# Register original (untrimmed) request details for manual resend
|
||
try:
|
||
pipeline_id = str((context.get("meta") or {}).get("id", "pipeline_editor"))
|
||
except Exception:
|
||
pipeline_id = "pipeline_editor"
|
||
try:
|
||
register_http_request(req_id, {
|
||
"pipeline_id": pipeline_id,
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"provider": provider,
|
||
"method": "POST",
|
||
"url": url,
|
||
"headers": dict(final_headers),
|
||
"body_json": payload, # untrimmed original payload
|
||
})
|
||
except Exception:
|
||
pass
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
await trace_fn({
|
||
"event": "http_req",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"provider": provider,
|
||
"req_id": req_id,
|
||
"method": "POST",
|
||
"url": url,
|
||
"headers": final_headers,
|
||
"body_text": json.dumps(_sanitize_b64_for_log(payload, max_len=180), ensure_ascii=False, indent=2),
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
|
||
# Timeout from run-meta (seconds); fallback to 60
|
||
try:
|
||
timeout_sec = float((context.get("meta") or {}).get("http_timeout_sec", 60) or 60)
|
||
except Exception:
|
||
timeout_sec = 60.0
|
||
|
||
st: Optional[int] = None
|
||
async with build_client(timeout=timeout_sec) as client:
|
||
body_bytes = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||
# Cooperative cancel pre-check (avoid starting new HTTP on abort)
|
||
try:
|
||
pipeline_id = str((context.get("meta") or {}).get("id", "pipeline_editor"))
|
||
except Exception:
|
||
pipeline_id = "pipeline_editor"
|
||
try:
|
||
if is_cancelled(pipeline_id) and get_cancel_mode(pipeline_id) == "abort":
|
||
try:
|
||
print(f"TRACE http_cancel_pre: {self.node_id} abort before request")
|
||
except Exception:
|
||
pass
|
||
raise ExecutionError("Cancelled by user (abort)")
|
||
except Exception:
|
||
pass
|
||
|
||
async def _do_post():
|
||
return await client.post(url, content=body_bytes, headers=final_headers)
|
||
|
||
# Await HTTP with cooperative cancel/abort handling
|
||
resp = await _await_coro_with_cancel(_do_post(), pipeline_id)
|
||
# Do not raise_for_status: keep body/logs on 4xx/5xx
|
||
try:
|
||
print("===== ProviderCall RESPONSE BEGIN =====")
|
||
print(f"node={self.node_id} type={self.type_name} provider={provider}")
|
||
print(f"Status: {resp.status_code}")
|
||
try:
|
||
print("Headers:")
|
||
print(json.dumps(dict(resp.headers), ensure_ascii=False, indent=2))
|
||
except Exception:
|
||
try:
|
||
print(f"Headers(raw): {dict(resp.headers)}")
|
||
except Exception:
|
||
print("Headers(raw): <unavailable>")
|
||
# Печатаем JSON тела с ТРИММИНГОМ ТОЛЬКО base64 строк, оставляя остальную структуру нетронутой
|
||
try:
|
||
raw_obj = _safe_response_json(resp)
|
||
import re as _re
|
||
def _is_b64ish(s: str) -> bool:
|
||
try:
|
||
if not isinstance(s, str) or len(s) < 128:
|
||
return False
|
||
# быстрый тест на base64 (разрешаем переводы строк)
|
||
return bool(_re.fullmatch(r"[A-Za-z0-9+/=\r\n]+", s))
|
||
except Exception:
|
||
return False
|
||
def _trim_b64_in_obj(x):
|
||
if isinstance(x, dict):
|
||
out = {}
|
||
for k, v in x.items():
|
||
if isinstance(v, str) and "data" in str(k).lower() and _is_b64ish(v) and len(v) > 256:
|
||
out[k] = v[:180] + f"... (trimmed {len(v)-180})"
|
||
else:
|
||
out[k] = _trim_b64_in_obj(v)
|
||
return out
|
||
if isinstance(x, list):
|
||
return [_trim_b64_in_obj(i) for i in x]
|
||
return x
|
||
sanitized_obj = _trim_b64_in_obj(raw_obj if isinstance(raw_obj, (dict, list)) else {})
|
||
body_text = json.dumps(sanitized_obj if sanitized_obj else raw_obj, ensure_ascii=False, indent=2)
|
||
except Exception:
|
||
body_text = _safe_response_text(resp)
|
||
print("Body JSON:")
|
||
print(body_text)
|
||
print("===== ProviderCall RESPONSE END =====")
|
||
except Exception:
|
||
body_text = "<resp.text decode error>"
|
||
pass
|
||
try:
|
||
data = _safe_response_json(resp)
|
||
except Exception:
|
||
data = {"error": "Failed to decode JSON from upstream", "text": _safe_response_text(resp)}
|
||
# Построим данные для панели Data:
|
||
# - images_data: полный список inline картинок (mime+base64, НЕ обрезанные)
|
||
# - data_preview: краткая сводка, если картинок нет или UI не может их отрисовать
|
||
try:
|
||
def _collect_images_http(obj: Any) -> List[Dict[str, str]]:
|
||
imgs: List[Dict[str, str]] = []
|
||
def walk(o: Any) -> None:
|
||
if isinstance(o, dict):
|
||
# Gemini: inlineData / inline_data
|
||
if isinstance(o.get("inlineData"), dict):
|
||
idat = o.get("inlineData") or {}
|
||
b64 = str(idat.get("data") or "")
|
||
mime = str(idat.get("mimeType") or idat.get("mime_type") or "image/png")
|
||
if b64:
|
||
imgs.append({"mime": mime, "data": b64})
|
||
if isinstance(o.get("inline_data"), dict):
|
||
idat = o.get("inline_data") or {}
|
||
b64 = str(idat.get("data") or "")
|
||
mime = str(idat.get("mimeType") or idat.get("mime_type") or "image/png")
|
||
if b64:
|
||
imgs.append({"mime": mime, "data": b64})
|
||
# Claude style
|
||
if (o.get("type") == "image") and isinstance(o.get("source"), dict):
|
||
src = o.get("source") or {}
|
||
if (src.get("type") or src.get("source_type")) in {"base64", "inline"}:
|
||
b64 = str(src.get("data") or "")
|
||
mime = str(src.get("media_type") or src.get("mediaType") or "image/png")
|
||
if b64:
|
||
imgs.append({"mime": mime, "data": b64})
|
||
for v in o.values():
|
||
walk(v)
|
||
elif isinstance(o, list):
|
||
for v in o:
|
||
walk(v)
|
||
walk(obj)
|
||
return imgs
|
||
|
||
def _collect_image_summaries_http(obj: Any) -> List[str]:
|
||
info: List[str] = []
|
||
def walk(o: Any, path: str = "") -> None:
|
||
if isinstance(o, dict):
|
||
if isinstance(o.get("inlineData"), dict):
|
||
idat = o.get("inlineData") or {}
|
||
b64 = str(idat.get("data") or "")
|
||
mime = str(idat.get("mimeType") or idat.get("mime_type") or "")
|
||
info.append(f"{path}/inlineData mime={mime or '?'} b64_len={len(b64)}")
|
||
if isinstance(o.get("inline_data"), dict):
|
||
idat = o.get("inline_data") or {}
|
||
b64 = str(idat.get("data") or "")
|
||
mime = str(idat.get("mimeType") or idat.get("mime_type") or "")
|
||
info.append(f"{path}/inline_data mime={mime or '?'} b64_len={len(b64)}")
|
||
if (o.get("type") == "image") and isinstance(o.get("source"), dict):
|
||
src = o.get("source") or {}
|
||
if (src.get("type") or src.get("source_type")) in {"base64", "inline"}:
|
||
b64 = str(src.get("data") or "")
|
||
mime = str(src.get("media_type") or src.get("mediaType") or "")
|
||
info.append(f"{path}/source mime={mime or '?'} b64_len={len(b64)}")
|
||
for k, v in o.items():
|
||
walk(v, path + (("." + k) if path else k))
|
||
elif isinstance(o, list):
|
||
for i, v in enumerate(o):
|
||
walk(v, path + f"[{i}]")
|
||
walk(obj, "")
|
||
return info
|
||
|
||
images_data = _collect_images_http(data)
|
||
_img_lines_http = _collect_image_summaries_http(data)
|
||
data_preview = ""
|
||
if _img_lines_http:
|
||
data_preview = "images: " + str(len(_img_lines_http)) + "\n" + "\n".join(_img_lines_http[:12])
|
||
except Exception:
|
||
images_data = []
|
||
data_preview = ""
|
||
# SSE: http_resp
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
await trace_fn({
|
||
"event": "http_resp",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"provider": provider,
|
||
"req_id": req_id,
|
||
"status": int(resp.status_code),
|
||
"headers": dict(resp.headers),
|
||
"body_text": body_text if isinstance(body_text, str) else str(body_text),
|
||
"data_preview": data_preview,
|
||
"images": images_data,
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
try:
|
||
st = int(resp.status_code)
|
||
except Exception:
|
||
st = None
|
||
|
||
# Извлечение текста: приоритет — пресет ноды -> кастомн. поля ноды -> глобальные meta
|
||
try:
|
||
meta_opts = (context.get("meta") or {})
|
||
presets = meta_opts.get("text_extract_presets") or []
|
||
cfg = self.config or {}
|
||
|
||
strategy = None
|
||
json_path = None
|
||
join_sep = None
|
||
|
||
# 1) Пресет по id
|
||
pid = str(cfg.get("text_extract_preset_id") or "").strip()
|
||
if pid:
|
||
try:
|
||
pr = next((x for x in presets if isinstance(x, dict) and str(x.get("id", "")) == pid), None)
|
||
except Exception:
|
||
pr = None
|
||
if pr:
|
||
strategy = str(pr.get("strategy") or "auto")
|
||
json_path = str(pr.get("json_path") or "")
|
||
join_sep = str(pr.get("join_sep") or "\n")
|
||
|
||
# 2) Кастомные поля ноды (если задано что-то из набора)
|
||
if not strategy and (cfg.get("text_extract_strategy") is not None or cfg.get("text_extract_json_path") is not None or cfg.get("text_join_sep") is not None):
|
||
strategy = str(cfg.get("text_extract_strategy") or "auto")
|
||
json_path = str(cfg.get("text_extract_json_path") or "")
|
||
join_sep = str(cfg.get("text_join_sep") or "\n")
|
||
|
||
# 3) Глобальные метаданные запуска
|
||
if not strategy:
|
||
strategy = str(meta_opts.get("text_extract_strategy", "auto") or "auto")
|
||
json_path = str(meta_opts.get("text_extract_json_path", "") or "")
|
||
join_sep = str(meta_opts.get("text_join_sep", "\n") or "\n")
|
||
except Exception:
|
||
strategy, json_path, join_sep = "auto", "", "\n"
|
||
|
||
text = _extract_text_for_out(data, strategy, provider, json_path, join_sep)
|
||
# Если текст пуст (например, пришли только inlineData/base64-картинки) — сформируем краткое резюме картинок,
|
||
# чтобы панель Data не оставалась пустой.
|
||
if not text:
|
||
try:
|
||
def _collect_image_summaries(obj: Any) -> List[str]:
|
||
info: List[str] = []
|
||
def walk(o: Any, path: str = "") -> None:
|
||
if isinstance(o, dict):
|
||
# Gemini: inlineData / inline_data
|
||
if isinstance(o.get("inlineData"), dict):
|
||
idat = o.get("inlineData") or {}
|
||
b64 = str(idat.get("data") or "")
|
||
mime = str(idat.get("mimeType") or idat.get("mime_type") or "")
|
||
info.append(f"{path}/inlineData mime={mime or '?'} b64_len={len(b64)}")
|
||
if isinstance(o.get("inline_data"), dict):
|
||
idat = o.get("inline_data") or {}
|
||
b64 = str(idat.get("data") or "")
|
||
mime = str(idat.get("mimeType") or idat.get("mime_type") or "")
|
||
info.append(f"{path}/inline_data mime={mime or '?'} b64_len={len(b64)}")
|
||
# Claude: {"type":"image","source":{"type":"base64","media_type":..., "data":...}}
|
||
if (o.get("type") == "image") and isinstance(o.get("source"), dict):
|
||
src = o.get("source") or {}
|
||
if (src.get("type") or src.get("source_type")) in {"base64", "inline"}:
|
||
b64 = str(src.get("data") or "")
|
||
mime = str(src.get("media_type") or src.get("mediaType") or "")
|
||
info.append(f"{path}/source mime={mime or '?'} b64_len={len(b64)}")
|
||
for k, v in o.items():
|
||
walk(v, path + (("." + k) if path else k))
|
||
elif isinstance(o, list):
|
||
for i, v in enumerate(o):
|
||
walk(v, path + f"[{i}]")
|
||
walk(obj, "")
|
||
return info
|
||
img_lines = _collect_image_summaries(data)
|
||
if img_lines:
|
||
text = "images: " + str(len(img_lines)) + "\n" + "\n".join(img_lines[:12])
|
||
except Exception:
|
||
pass
|
||
# Подробный SSE-лог завершения ProviderCall
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
# Compute destination macros for extracted text
|
||
try:
|
||
to_path = f"OUT.{self.node_id}.response_text"
|
||
macro_braces = f"{{{{ OUT.{self.node_id}.response_text }}}}"
|
||
alias_macro = ""
|
||
m = re.match(r"^n(\d+)$", str(self.node_id))
|
||
if m:
|
||
alias_macro = f"[[OUT{int(m.group(1))}]]"
|
||
except Exception:
|
||
to_path = f"OUT.{self.node_id}.response_text"
|
||
macro_braces = f"{{{{ OUT.{self.node_id}.response_text }}}}"
|
||
alias_macro = ""
|
||
await trace_fn({
|
||
"event": "provider_done",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"provider": provider,
|
||
"url": url,
|
||
"req_id": req_id,
|
||
"status": int(st) if st is not None else None,
|
||
"text_len": int(len(text or "")),
|
||
"extracted_text": text or "",
|
||
"strategy": strategy,
|
||
"json_path": json_path or "",
|
||
"join_sep": join_sep or "\n",
|
||
"to_path": to_path,
|
||
"to_macro_outx": alias_macro,
|
||
"to_macro_braces": macro_braces,
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
return {"result": data, "response_text": text or ""}
|
||
|
||
|
||
class RawForwardNode(Node):
|
||
type_name = "RawForward"
|
||
|
||
async def run(self, inputs: Dict[str, Any], context: Dict[str, Any]) -> Dict[str, Any]:
|
||
"""
|
||
Прямой форвард входящего запроса на апстрим:
|
||
- base_url/override_path/extra_headers поддерживают макросы [[...]] и {{ ... }}
|
||
- По умолчанию пробрасываем входящие заголовки (кроме Host/Content-Length), затем применяем extra_headers поверх.
|
||
- Тело берём из incoming.json (если есть), иначе пытаемся использовать текст/байты.
|
||
- Эмитим SSE события http_req/http_resp для панели логов.
|
||
"""
|
||
# While-loop wrapper: if while_expr is set or ignore_errors enabled, route through helper (single-run when expr empty)
|
||
try:
|
||
cfg = self.config or {}
|
||
except Exception:
|
||
cfg = {}
|
||
try:
|
||
_while_expr = str(cfg.get("while_expr") or "").strip()
|
||
except Exception:
|
||
_while_expr = ""
|
||
try:
|
||
_ignore = bool(cfg.get("ignore_errors", False))
|
||
except Exception:
|
||
_ignore = False
|
||
if not (context.get("_in_while") or False) and (_while_expr or _ignore):
|
||
return await _rawforward_run_with_while(self, inputs, context)
|
||
|
||
incoming = context.get("incoming", {}) or {}
|
||
raw_payload = incoming.get("json")
|
||
|
||
# Optional sleep before forwarding (UI-configured, milliseconds)
|
||
try:
|
||
sleep_ms = int(self.config.get("sleep_ms") or 0)
|
||
except Exception:
|
||
sleep_ms = 0
|
||
if sleep_ms > 0:
|
||
# Notify UI to show "sleep" state for this node
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
await trace_fn({
|
||
"event": "node_sleep",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"sleep_ms": int(sleep_ms),
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
try:
|
||
await asyncio.sleep(max(0.0, sleep_ms / 1000.0))
|
||
except Exception:
|
||
pass
|
||
|
||
# Конфиг с поддержкой макросов
|
||
base_url: Optional[str] = self.config.get("base_url")
|
||
override_path: Optional[str] = self.config.get("override_path")
|
||
|
||
out_map_for_macros = context.get("OUT") or {}
|
||
if base_url:
|
||
base_url = render_template_simple(str(base_url), context, out_map_for_macros)
|
||
if override_path:
|
||
override_path = render_template_simple(str(override_path), context, out_map_for_macros)
|
||
|
||
# Автодетекция вендора для базового URL если base_url не задан
|
||
if not base_url:
|
||
vendor = detect_vendor(raw_payload)
|
||
base_url = None
|
||
try:
|
||
base_url = _adapter_default_base_url_for(vendor)
|
||
except Exception:
|
||
base_url = None
|
||
if not base_url:
|
||
if vendor == "openai":
|
||
base_url = "https://api.openai.com"
|
||
elif vendor == "claude":
|
||
base_url = "https://api.anthropic.com"
|
||
elif vendor == "gemini":
|
||
base_url = "https://generativelanguage.googleapis.com"
|
||
if not base_url:
|
||
raise ExecutionError(
|
||
f"Node {self.node_id} ({self.type_name}): 'base_url' is not configured and vendor could not be detected."
|
||
)
|
||
|
||
if not str(base_url).startswith(("http://", "https://")):
|
||
base_url = "http://" + str(base_url)
|
||
|
||
path = override_path or incoming.get("path") or "/"
|
||
query = incoming.get("query")
|
||
path_with_qs = f"{path}?{query}" if query else str(path)
|
||
url = urljoin(str(base_url).rstrip("/") + "/", str(path_with_qs).lstrip("/"))
|
||
|
||
# Заголовки: passthrough + extra_headers(JSON)
|
||
passthrough_headers: bool = bool(self.config.get("passthrough_headers", True))
|
||
extra_headers_json: str = self.config.get("extra_headers") or "{}"
|
||
try:
|
||
extra_headers_src = render_template_simple(extra_headers_json, context, out_map_for_macros) if extra_headers_json else "{}"
|
||
extra_headers = json.loads(extra_headers_src) if extra_headers_src else {}
|
||
if not isinstance(extra_headers, dict):
|
||
raise ValueError("extra_headers must be an object")
|
||
except Exception as exc: # noqa: BLE001
|
||
raise ExecutionError(f"RawForward extra_headers invalid JSON: {exc}")
|
||
|
||
headers: Dict[str, str] = {}
|
||
if passthrough_headers:
|
||
inc_headers = incoming.get("headers") or {}
|
||
for k, v in inc_headers.items():
|
||
# Не пробрасываем управляющие заголовки, которые рассчитает httpx/сервер
|
||
if k and k.lower() not in {"host", "content-length"}:
|
||
headers[k] = v
|
||
# extra_headers перекрывают проброс
|
||
try:
|
||
headers.update(extra_headers)
|
||
except Exception:
|
||
pass
|
||
# Нормализация: защитимся от повторного наличия host/content-length после update
|
||
for k in list(headers.keys()):
|
||
kl = k.lower()
|
||
if kl in {"host", "content-length"}:
|
||
try:
|
||
headers.pop(k, None)
|
||
except Exception:
|
||
pass
|
||
|
||
# Метод
|
||
method = str(incoming.get("method") or "POST").upper().strip() or "POST"
|
||
|
||
# Тело запроса
|
||
body_text = ""
|
||
body_bytes: bytes = b""
|
||
if raw_payload is not None:
|
||
# JSON — сериализуем явно как UTF‑8
|
||
try:
|
||
body_text = json.dumps(raw_payload, ensure_ascii=False, indent=2)
|
||
except Exception:
|
||
try:
|
||
body_text = str(raw_payload)
|
||
except Exception:
|
||
body_text = ""
|
||
try:
|
||
body_bytes = json.dumps(raw_payload, ensure_ascii=False).encode("utf-8")
|
||
except Exception:
|
||
body_bytes = (body_text or "").encode("utf-8", errors="ignore")
|
||
# Если Content-Type не задан — выставим JSON
|
||
if "content-type" not in {k.lower() for k in headers.keys()}:
|
||
headers["Content-Type"] = "application/json"
|
||
else:
|
||
# Попробуем текст/байты из incoming
|
||
raw_bytes = incoming.get("body_bytes") or incoming.get("bytes")
|
||
raw_text = incoming.get("body_text") or incoming.get("text")
|
||
if isinstance(raw_bytes, (bytes, bytearray)):
|
||
body_bytes = bytes(raw_bytes)
|
||
try:
|
||
body_text = body_bytes.decode("utf-8", errors="ignore")
|
||
except Exception:
|
||
body_text = ""
|
||
elif raw_text is not None:
|
||
body_text = str(raw_text)
|
||
try:
|
||
body_bytes = body_text.encode("utf-8")
|
||
except Exception:
|
||
body_bytes = (body_text or "").encode("utf-8", errors="ignore")
|
||
else:
|
||
body_text = ""
|
||
body_bytes = b""
|
||
|
||
# Таймаут из метаданных запуска
|
||
try:
|
||
timeout_sec = float((context.get("meta") or {}).get("http_timeout_sec", 60) or 60)
|
||
except Exception:
|
||
timeout_sec = 60.0
|
||
|
||
# Диагностический вывод полной заявки/ответа (request body sanitized)
|
||
try:
|
||
print("===== RawForward REQUEST BEGIN =====")
|
||
print(f"node={self.node_id} type={self.type_name}")
|
||
print(f"Method: {method}")
|
||
print(f"URL: {url}")
|
||
try:
|
||
print("Headers:")
|
||
print(json.dumps(headers, ensure_ascii=False, indent=2))
|
||
except Exception:
|
||
print(f"Headers(raw): {headers}")
|
||
print("Body:")
|
||
try:
|
||
san_body_text = _sanitize_json_string_for_log(body_text, max_len=180)
|
||
except Exception:
|
||
san_body_text = body_text
|
||
print(san_body_text)
|
||
print("===== RawForward REQUEST END =====")
|
||
except Exception:
|
||
pass
|
||
|
||
# SSE: http_req
|
||
req_id = f"{self.node_id}-{int(time.time()*1000)}"
|
||
# Register original (untrimmed) request details for manual resend
|
||
try:
|
||
pipeline_id = str((context.get("meta") or {}).get("id", "pipeline_editor"))
|
||
except Exception:
|
||
pipeline_id = "pipeline_editor"
|
||
try:
|
||
reg_info: Dict[str, Any] = {
|
||
"pipeline_id": pipeline_id,
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"method": method,
|
||
"url": url,
|
||
"headers": dict(headers),
|
||
}
|
||
if raw_payload is not None:
|
||
reg_info["body_json"] = raw_payload # original JSON body
|
||
# Always keep original textual/bytes preview for non-JSON
|
||
reg_info["body_text"] = body_text
|
||
try:
|
||
reg_info["body_bytes_len"] = int(len(body_bytes or b""))
|
||
except Exception:
|
||
pass
|
||
register_http_request(req_id, reg_info)
|
||
except Exception:
|
||
pass
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
await trace_fn({
|
||
"event": "http_req",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"req_id": req_id,
|
||
"method": method,
|
||
"url": url,
|
||
"headers": headers,
|
||
"body_text": san_body_text if isinstance(san_body_text, str) else body_text,
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
|
||
# Отправка
|
||
async with build_client(timeout=timeout_sec) as client:
|
||
# Для GET/HEAD обычно не отправляем body
|
||
send_content = None if method in {"GET", "HEAD"} else body_bytes
|
||
# Cooperative cancel pre-check (avoid starting new HTTP on abort)
|
||
try:
|
||
pipeline_id = str((context.get("meta") or {}).get("id", "pipeline_editor"))
|
||
except Exception:
|
||
pipeline_id = "pipeline_editor"
|
||
try:
|
||
if is_cancelled(pipeline_id) and get_cancel_mode(pipeline_id) == "abort":
|
||
try:
|
||
print(f"TRACE http_cancel_pre: {self.node_id} abort before request")
|
||
except Exception:
|
||
pass
|
||
raise ExecutionError("Cancelled by user (abort)")
|
||
except Exception:
|
||
pass
|
||
|
||
async def _do_req():
|
||
return await client.request(method, url, headers=headers, content=send_content)
|
||
|
||
# Await HTTP with cooperative cancel/abort handling
|
||
resp = await _await_coro_with_cancel(_do_req(), pipeline_id)
|
||
|
||
# Ответ: лог/печать
|
||
try:
|
||
print("===== RawForward RESPONSE BEGIN =====")
|
||
print(f"node={self.node_id} type={self.type_name}")
|
||
print(f"Status: {resp.status_code}")
|
||
try:
|
||
print("Headers:")
|
||
print(json.dumps(dict(resp.headers), ensure_ascii=False, indent=2))
|
||
except Exception:
|
||
try:
|
||
print(f"Headers(raw): {dict(resp.headers)}")
|
||
except Exception:
|
||
print("Headers(raw): <unavailable>")
|
||
try:
|
||
resp_text = _safe_response_text(resp)
|
||
except Exception:
|
||
resp_text = "<resp.text decode error>"
|
||
print("Body Text:")
|
||
print(resp_text)
|
||
print("===== RawForward RESPONSE END =====")
|
||
except Exception:
|
||
resp_text = "<resp.text decode error>"
|
||
|
||
# SSE: http_resp
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
await trace_fn({
|
||
"event": "http_resp",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"req_id": req_id,
|
||
"status": int(resp.status_code),
|
||
"headers": dict(resp.headers),
|
||
"body_text": resp_text if isinstance(resp_text, str) else str(resp_text),
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
|
||
# Разбор результата
|
||
try:
|
||
data = _safe_response_json(resp)
|
||
except Exception:
|
||
data = {"error": "Failed to decode JSON from upstream", "text": resp_text}
|
||
|
||
# Извлечение текста: приоритет — пресет ноды -> кастомные поля ноды -> глобальные meta.
|
||
try:
|
||
meta_opts = (context.get("meta") or {})
|
||
presets = meta_opts.get("text_extract_presets") or []
|
||
cfg = self.config or {}
|
||
|
||
strategy = None
|
||
json_path = None
|
||
join_sep = None
|
||
|
||
# 1) Пресет по id, выбранный в ноде
|
||
pid = str(cfg.get("text_extract_preset_id") or "").strip()
|
||
if pid:
|
||
try:
|
||
pr = next((x for x in presets if isinstance(x, dict) and str(x.get("id", "")) == pid), None)
|
||
except Exception:
|
||
pr = None
|
||
if pr:
|
||
strategy = str(pr.get("strategy") or "auto")
|
||
json_path = str(pr.get("json_path") or "")
|
||
join_sep = str(pr.get("join_sep") or "\n")
|
||
|
||
# 2) Кастомные поля ноды (если что-то задано)
|
||
if not strategy and (cfg.get("text_extract_strategy") is not None or cfg.get("text_extract_json_path") is not None or cfg.get("text_join_sep") is not None):
|
||
strategy = str(cfg.get("text_extract_strategy") or "auto")
|
||
json_path = str(cfg.get("text_extract_json_path") or "")
|
||
join_sep = str(cfg.get("text_join_sep") or "\n")
|
||
|
||
# 3) Глобальные метаданные запуска (дефолт)
|
||
if not strategy:
|
||
strategy = str(meta_opts.get("text_extract_strategy", "auto") or "auto")
|
||
json_path = str(meta_opts.get("text_extract_json_path", "") or "")
|
||
join_sep = str(meta_opts.get("text_join_sep", "\n") or "\n")
|
||
except Exception:
|
||
strategy, json_path, join_sep = "auto", "", "\n"
|
||
|
||
# Подсказка о провайдере: используем detect_vendor(data) как hint для auto
|
||
try:
|
||
prov_hint = detect_vendor(data if isinstance(data, dict) else {}) # type: ignore[arg-type]
|
||
except Exception:
|
||
prov_hint = "unknown"
|
||
|
||
best_text = _extract_text_for_out(data, strategy, prov_hint, json_path, join_sep)
|
||
# Подробный SSE-лог завершения RawForward
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
# Compute destination macros for extracted text and include req_id
|
||
try:
|
||
to_path = f"OUT.{self.node_id}.response_text"
|
||
macro_braces = f"{{{{ OUT.{self.node_id}.response_text }}}}"
|
||
alias_macro = ""
|
||
m = re.match(r"^n(\d+)$", str(self.node_id))
|
||
if m:
|
||
alias_macro = f"[[OUT{int(m.group(1))}]]"
|
||
except Exception:
|
||
to_path = f"OUT.{self.node_id}.response_text"
|
||
macro_braces = f"{{{{ OUT.{self.node_id}.response_text }}}}"
|
||
alias_macro = ""
|
||
await trace_fn({
|
||
"event": "rawforward_done",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"method": method,
|
||
"url": url,
|
||
"req_id": req_id,
|
||
"status": int(resp.status_code) if 'resp' in locals() else None,
|
||
"text_len": int(len(best_text or "")),
|
||
"extracted_text": best_text or "",
|
||
"strategy": strategy,
|
||
"json_path": json_path or "",
|
||
"join_sep": join_sep or "\n",
|
||
"to_path": to_path,
|
||
"to_macro_outx": alias_macro,
|
||
"to_macro_braces": macro_braces,
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
return {"result": data, "response_text": best_text or ""}
|
||
|
||
class ReturnNode(Node):
|
||
type_name = "Return"
|
||
|
||
async def run(self, inputs: Dict[str, Any], context: Dict[str, Any]) -> Dict[str, Any]: # noqa: D401
|
||
# Определяем целевой формат
|
||
cfg = self.config or {}
|
||
target = str(cfg.get("target_format", "auto")).lower().strip() or "auto"
|
||
if target == "auto":
|
||
target = str(context.get("vendor_format") or "openai").lower().strip() or "openai"
|
||
|
||
# Рендерим текст из шаблона (по умолчанию берём [[OUT1]])
|
||
out_map = context.get("OUT") or {}
|
||
template = cfg.get("text_template")
|
||
if template is None or template == "":
|
||
template = "[[OUT1]]"
|
||
try:
|
||
text = render_template_simple(str(template), context, out_map)
|
||
except Exception:
|
||
text = ""
|
||
|
||
model = str(context.get("model") or "")
|
||
|
||
# Форматтеры под провайдеры (как в execute_pipeline_echo)
|
||
def fmt_openai(t: str) -> Dict[str, Any]:
|
||
return {
|
||
"id": "ret_mock_123",
|
||
"object": "chat.completion",
|
||
"model": model,
|
||
"choices": [
|
||
{
|
||
"index": 0,
|
||
"message": {"role": "assistant", "content": t},
|
||
"finish_reason": "stop",
|
||
}
|
||
],
|
||
"usage": {"prompt_tokens": 0, "completion_tokens": len((t or "").split()), "total_tokens": 0},
|
||
}
|
||
|
||
def fmt_gemini(t: str) -> Dict[str, Any]:
|
||
return {
|
||
"candidates": [
|
||
{
|
||
"content": {
|
||
"role": "model",
|
||
"parts": [{"text": t}],
|
||
},
|
||
"finishReason": "STOP",
|
||
"index": 0,
|
||
}
|
||
],
|
||
"modelVersion": model,
|
||
}
|
||
|
||
def fmt_claude(t: str) -> Dict[str, Any]:
|
||
return {
|
||
"id": "msg_ret_123",
|
||
"type": "message",
|
||
"model": model,
|
||
"role": "assistant",
|
||
"content": [
|
||
{"type": "text", "text": t}
|
||
],
|
||
"stop_reason": "end_turn",
|
||
}
|
||
|
||
if target == "openai":
|
||
result = fmt_openai(text)
|
||
elif target == "gemini":
|
||
result = fmt_gemini(text)
|
||
elif target == "claude":
|
||
result = fmt_claude(text)
|
||
else:
|
||
# неизвестное значение — безопасный дефолт
|
||
result = fmt_openai(text)
|
||
|
||
# Подробный SSE-лог Return
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
await trace_fn({
|
||
"event": "return_detail",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"target": target,
|
||
"text_len": int(len(text or "")),
|
||
"template_used": str(template),
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
|
||
return {"result": result, "response_text": text}
|
||
|
||
|
||
class IfNode(Node):
|
||
type_name = "If"
|
||
|
||
async def run(self, inputs: Dict[str, Any], context: Dict[str, Any]) -> Dict[str, Any]: # noqa: D401
|
||
expr = str(self.config.get("expr") or "").strip()
|
||
out_map = context.get("OUT") or {}
|
||
# Для логов: отображаем развёрнутое выражение (с подставленными макросами)
|
||
try:
|
||
expanded = render_template_simple(expr, context, out_map)
|
||
except Exception:
|
||
expanded = expr
|
||
try:
|
||
res = bool(eval_condition_expr(expr, context, out_map)) if expr else False
|
||
except Exception as exc: # noqa: BLE001
|
||
# Расширенный лог, чтобы быстрее найти ошибку парсинга/оценки выражения
|
||
try:
|
||
print(f"TRACE if_error: {self.node_id} expr={expr!r} expanded={expanded!r} error={exc}")
|
||
except Exception:
|
||
pass
|
||
raise ExecutionError(f"If expr error: {exc}")
|
||
try:
|
||
print(f"TRACE if: {self.node_id} expr={expr!r} expanded={expanded!r} result={str(res).lower()}")
|
||
except Exception:
|
||
pass
|
||
# Подробный SSE-лог по If
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
await trace_fn({
|
||
"event": "if_result",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"expr": expr,
|
||
"expanded": expanded,
|
||
"result": bool(res),
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
return {"result": res, "true": res, "false": (not res)}
|
||
|
||
|
||
NODE_REGISTRY.update({
|
||
SetVarsNode.type_name: SetVarsNode,
|
||
ProviderCallNode.type_name: ProviderCallNode,
|
||
RawForwardNode.type_name: RawForwardNode,
|
||
ReturnNode.type_name: ReturnNode,
|
||
IfNode.type_name: IfNode,
|
||
})
|
||
|
||
|
||
|
||
|
||
# --- While-loop helpers for ProviderCall and RawForward (node-scoped) -----------
|
||
async def _providercall_run_with_while(self, inputs, context):
|
||
"""
|
||
Execute ProviderCall node in a while loop using the same If-expression parser.
|
||
- Config keys (optional):
|
||
while_expr: string expression (same grammar as If: &&, ||, !, contains, [[...]], {{ ... }})
|
||
while_max_iters: safety cap (default: 50)
|
||
ignore_errors: bool; if True, HTTP/client errors do not crash pipeline; result carries {"error": "..."}
|
||
- Local variables within loop:
|
||
[[cycleindex]]: 0-based iteration index (scoped to this node call)
|
||
[[WAS_ERROR]]: True only for current iteration when an exception occurred (scoped to this node call)
|
||
These locals are injected into context.vars for the inner single-run only and do not leak globally.
|
||
- Outputs (global user vars for downstream nodes):
|
||
[[WAS_ERROR__nX]]: bool — final iteration error flag for this node
|
||
[[CYCLEINDEX__nX]]: int — index of the last executed iteration for this node (0 if none executed)
|
||
"""
|
||
try:
|
||
cfg = self.config or {}
|
||
except Exception:
|
||
cfg = {}
|
||
# Expression and safety cap
|
||
try:
|
||
expr = str(cfg.get("while_expr") or "").strip()
|
||
except Exception:
|
||
expr = ""
|
||
try:
|
||
max_iters = int(cfg.get("while_max_iters") or 50)
|
||
except Exception:
|
||
max_iters = 50
|
||
if max_iters <= 0:
|
||
max_iters = 50
|
||
ignore = False
|
||
try:
|
||
ignore = bool(cfg.get("ignore_errors", False))
|
||
except Exception:
|
||
ignore = False
|
||
|
||
out_map = context.get("OUT") or {}
|
||
last_out = {"result": {}, "response_text": ""}
|
||
last_was_error = False
|
||
last_idx = -1
|
||
|
||
for i in range(max_iters):
|
||
# Cancel check before starting next iteration
|
||
try:
|
||
pid = str((context.get("meta") or {}).get("id", "pipeline_editor"))
|
||
except Exception:
|
||
pid = "pipeline_editor"
|
||
try:
|
||
if is_cancelled(pid):
|
||
mode = get_cancel_mode(pid)
|
||
try:
|
||
print(f"TRACE while_cancel: {self.node_id} mode={mode} at i={i} (pre)")
|
||
except Exception:
|
||
pass
|
||
break
|
||
except Exception:
|
||
pass
|
||
|
||
# Build loop-local context with cycleindex and WAS_ERROR
|
||
ctx2 = dict(context or {})
|
||
try:
|
||
vmap = dict(ctx2.get("vars") or {})
|
||
except Exception:
|
||
vmap = {}
|
||
vmap["cycleindex"] = i
|
||
vmap["WAS_ERROR"] = False # local flag for this iteration only
|
||
ctx2["vars"] = vmap
|
||
# mark reentry to avoid recursive while wrapping in self.run()
|
||
ctx2["_in_while"] = True
|
||
|
||
# Do-while semantics:
|
||
# - First iteration always runs.
|
||
# - For subsequent iterations, evaluate expression with last_out injected into OUT map.
|
||
expanded = ""
|
||
if not expr:
|
||
# No expression -> perform exactly one iteration
|
||
cond = (i == 0)
|
||
else:
|
||
if i == 0:
|
||
cond = True
|
||
else:
|
||
# Inject previous iteration error flag so [[WAS_ERROR]] in while_expr refers to the last iteration
|
||
try:
|
||
vmap2 = dict(ctx2.get("vars") or {})
|
||
except Exception:
|
||
vmap2 = {}
|
||
vmap2["WAS_ERROR"] = bool(last_was_error)
|
||
ctx2["vars"] = vmap2
|
||
# Augment OUT with the last output of this node so [[OUTn]] / [[OUT:nX...]] can see it
|
||
try:
|
||
out_aug = dict(out_map or {})
|
||
out_aug[self.node_id] = dict(last_out or {})
|
||
except Exception:
|
||
out_aug = out_map
|
||
try:
|
||
expanded = render_template_simple(expr, ctx2, out_aug)
|
||
except Exception:
|
||
expanded = expr
|
||
try:
|
||
cond = bool(eval_condition_expr(expr, ctx2, out_aug))
|
||
except Exception as exc: # noqa: BLE001
|
||
cond = False
|
||
try:
|
||
print(f"TRACE while_error: {self.node_id} expr={expr!r} expanded={expanded!r} error={exc}")
|
||
except Exception:
|
||
pass
|
||
# SSE: while_result (analogous to if_result)
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
await trace_fn({
|
||
"event": "while_result",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"expr": expr,
|
||
"expanded": expanded,
|
||
"result": bool(cond),
|
||
"index": i,
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
try:
|
||
print(f"TRACE while: {self.node_id} expr={expr!r} expanded={expanded!r} index={i} result={str(cond).lower()}")
|
||
except Exception:
|
||
pass
|
||
if not cond:
|
||
break
|
||
|
||
# Single iteration run; reenter node.run with _in_while flag set
|
||
try:
|
||
inner_out = await self.run(inputs, ctx2)
|
||
except BaseException as exc:
|
||
if ignore:
|
||
inner_out = {"result": {"error": str(exc)}, "response_text": ""}
|
||
try:
|
||
print(f"TRACE suppressed_error(inner): {self.node_id} {exc}")
|
||
except Exception:
|
||
pass
|
||
else:
|
||
raise
|
||
|
||
# Detect error by sentinel on result
|
||
was_err = False
|
||
try:
|
||
r = inner_out.get("result")
|
||
was_err = bool(isinstance(r, dict) and ("error" in r))
|
||
except Exception:
|
||
was_err = False
|
||
|
||
last_out = dict(inner_out or {})
|
||
last_was_error = was_err
|
||
last_idx = i
|
||
|
||
# Finalize outputs with per-node global vars
|
||
try:
|
||
cyc = int(last_idx if last_idx >= 0 else 0)
|
||
except Exception:
|
||
cyc = 0
|
||
try:
|
||
last_out = dict(last_out or {})
|
||
except Exception:
|
||
last_out = {"result": {}, "response_text": ""}
|
||
|
||
try:
|
||
last_out["vars"] = {
|
||
f"WAS_ERROR__{self.node_id}": bool(last_was_error),
|
||
f"CYCLEINDEX__{self.node_id}": cyc,
|
||
}
|
||
except Exception:
|
||
# ensure at least WAS_ERROR var
|
||
last_out["vars"] = {f"WAS_ERROR__{self.node_id}": bool(last_was_error)}
|
||
return last_out
|
||
|
||
|
||
async def _rawforward_run_with_while(self, inputs, context):
|
||
"""
|
||
Execute RawForward node in a while loop using the same If-expression parser.
|
||
Conventions identical to _providercall_run_with_while() regarding config and variables.
|
||
"""
|
||
try:
|
||
cfg = self.config or {}
|
||
except Exception:
|
||
cfg = {}
|
||
try:
|
||
expr = str(cfg.get("while_expr") or "").strip()
|
||
except Exception:
|
||
expr = ""
|
||
try:
|
||
max_iters = int(cfg.get("while_max_iters") or 50)
|
||
except Exception:
|
||
max_iters = 50
|
||
if max_iters <= 0:
|
||
max_iters = 50
|
||
ignore = False
|
||
try:
|
||
ignore = bool(cfg.get("ignore_errors", False))
|
||
except Exception:
|
||
ignore = False
|
||
|
||
out_map = context.get("OUT") or {}
|
||
last_out = {"result": {}, "response_text": ""}
|
||
last_was_error = False
|
||
last_idx = -1
|
||
|
||
for i in range(max_iters):
|
||
# Cancel check before starting next iteration
|
||
try:
|
||
pid = str((context.get("meta") or {}).get("id", "pipeline_editor"))
|
||
except Exception:
|
||
pid = "pipeline_editor"
|
||
try:
|
||
if is_cancelled(pid):
|
||
mode = get_cancel_mode(pid)
|
||
try:
|
||
print(f"TRACE while_cancel: {self.node_id} mode={mode} at i={i} (pre)")
|
||
except Exception:
|
||
pass
|
||
break
|
||
except Exception:
|
||
pass
|
||
|
||
ctx2 = dict(context or {})
|
||
try:
|
||
vmap = dict(ctx2.get("vars") or {})
|
||
except Exception:
|
||
vmap = {}
|
||
vmap["cycleindex"] = i
|
||
vmap["WAS_ERROR"] = False
|
||
ctx2["vars"] = vmap
|
||
ctx2["_in_while"] = True
|
||
|
||
# Do-while semantics + last_out in OUT for subsequent checks
|
||
expanded = ""
|
||
if not expr:
|
||
cond = (i == 0)
|
||
else:
|
||
if i == 0:
|
||
cond = True
|
||
else:
|
||
# Inject previous iteration error flag so [[WAS_ERROR]] in while_expr refers to the last iteration
|
||
try:
|
||
vmap2 = dict(ctx2.get("vars") or {})
|
||
except Exception:
|
||
vmap2 = {}
|
||
vmap2["WAS_ERROR"] = bool(last_was_error)
|
||
ctx2["vars"] = vmap2
|
||
try:
|
||
out_aug = dict(out_map or {})
|
||
out_aug[self.node_id] = dict(last_out or {})
|
||
except Exception:
|
||
out_aug = out_map
|
||
try:
|
||
expanded = render_template_simple(expr, ctx2, out_aug)
|
||
except Exception:
|
||
expanded = expr
|
||
try:
|
||
cond = bool(eval_condition_expr(expr, ctx2, out_aug))
|
||
except Exception as exc: # noqa: BLE001
|
||
cond = False
|
||
try:
|
||
print(f"TRACE while_error: {self.node_id} expr={expr!r} expanded={expanded!r} error={exc}")
|
||
except Exception:
|
||
pass
|
||
# SSE: while_result (analogous to if_result)
|
||
try:
|
||
trace_fn = context.get("_trace")
|
||
if trace_fn:
|
||
await trace_fn({
|
||
"event": "while_result",
|
||
"node_id": self.node_id,
|
||
"node_type": self.type_name,
|
||
"expr": expr,
|
||
"expanded": expanded,
|
||
"result": bool(cond),
|
||
"index": i,
|
||
"ts": int(time.time() * 1000),
|
||
})
|
||
except Exception:
|
||
pass
|
||
try:
|
||
print(f"TRACE while: {self.node_id} expr={expr!r} expanded={expanded!r} index={i} result={str(cond).lower()}")
|
||
except Exception:
|
||
pass
|
||
if not cond:
|
||
break
|
||
|
||
try:
|
||
inner_out = await self.run(inputs, ctx2)
|
||
except BaseException as exc:
|
||
if ignore:
|
||
inner_out = {"result": {"error": str(exc)}, "response_text": ""}
|
||
try:
|
||
print(f"TRACE suppressed_error(inner): {self.node_id} {exc}")
|
||
except Exception:
|
||
pass
|
||
else:
|
||
raise
|
||
|
||
was_err = False
|
||
try:
|
||
r = inner_out.get("result")
|
||
was_err = bool(isinstance(r, dict) and ("error" in r))
|
||
except Exception:
|
||
was_err = False
|
||
|
||
last_out = dict(inner_out or {})
|
||
last_was_error = was_err
|
||
last_idx = i
|
||
|
||
try:
|
||
cyc = int(last_idx if last_idx >= 0 else 0)
|
||
except Exception:
|
||
cyc = 0
|
||
try:
|
||
last_out = dict(last_out or {})
|
||
except Exception:
|
||
last_out = {"result": {}, "response_text": ""}
|
||
|
||
try:
|
||
last_out["vars"] = {
|
||
f"WAS_ERROR__{self.node_id}": bool(last_was_error),
|
||
f"CYCLEINDEX__{self.node_id}": cyc,
|
||
}
|
||
except Exception:
|
||
last_out["vars"] = {f"WAS_ERROR__{self.node_id}": bool(last_was_error)}
|
||
return last_out |