@@ -2479,7 +2479,11 @@ class BigQueryLoggerConfig:
24792479 max_content_length: Max length for text content before truncation.
24802480 table_id: BigQuery table ID.
24812481 clustering_fields: Fields to cluster the table by.
2482- log_multi_modal_content: Whether to log detailed content parts.
2482+ log_multi_modal_content: Whether to log detailed content parts in
2483+ the ``content_parts`` column. When ``False``, ``content_parts`` is
2484+ written as ``[]`` and GCS offloading (``gcs_bucket_name``) is disabled
2485+ so large text remains inline (truncated at ``max_content_length``) in
2486+ ``content`` and binary parts are not stored.
24832487 retry_config: Retry configuration for writes.
24842488 batch_size: Number of rows per batch.
24852489 batch_flush_interval: Max time to wait before flushing a batch.
@@ -2521,7 +2525,13 @@ class BigQueryLoggerConfig:
25212525 SDK queries such as ``ERROR_SQL_PREDICATE`` treat any non-null
25222526 ``error_message`` as an error, so such a row still marks its session
25232527 ``has_error``.
2524- gcs_bucket_name: GCS bucket for offloading large content.
2528+ gcs_bucket_name: GCS bucket for offloading large text and binary content.
2529+ Offloading only runs when ``content_parts`` is persisted
2530+ (``log_multi_modal_content=True`` and ``"content_parts"`` not in
2531+ ``payload_column_denylist``); otherwise ``GCSOffloader`` is not
2532+ created, a warning is logged once per plugin instance, large text is
2533+ kept inline (truncated) in ``content``, and binary parts are not
2534+ stored.
25252535 connection_id: BigQuery connection ID for ObjectRef columns.
25262536 log_session_metadata: Whether to log session metadata.
25272537 custom_tags: Static custom tags to attach to every event.
@@ -2745,7 +2755,8 @@ class BigQueryLoggerConfig:
27452755 shutdown_timeout : float = 10.0
27462756 queue_max_size : int = 10000
27472757 content_formatter : Optional [Callable [[Any , str ], Any ]] = None
2748- # If provided, large content (images, audio, video, large text) will be offloaded to this GCS bucket.
2758+ # If provided and content_parts is persisted, large content (images, audio,
2759+ # video, large text) is offloaded to this GCS bucket.
27492760 gcs_bucket_name : Optional [str ] = None
27502761 # If provided, this connection ID will be used as the authorizer for ObjectRef columns.
27512762 # Format: "location.connection_id" (e.g. "us.my-connection")
@@ -4257,7 +4268,13 @@ def _upload_sync(
42574268
42584269
42594270class HybridContentParser :
4260- """Parses content and offloads large/binary parts to GCS."""
4271+ """Parses content and offloads large/binary parts to GCS.
4272+
4273+ GCS offloading runs when an offloader is configured and ``offload=True``
4274+ (the default). When ``offload=False``, binary parts stay inline with
4275+ ``"[BINARY DATA]"`` and oversized text parts stay inline (truncated at
4276+ ``max_length``) regardless of whether an offloader is present.
4277+ """
42614278
42624279 def __init__ (
42634280 self ,
@@ -4474,6 +4491,7 @@ async def _parse_content_object(
44744491 span_id : Optional [str ] = None ,
44754492 parse_uid : str = "" ,
44764493 content_ordinal : int = 0 ,
4494+ offload : bool = True ,
44774495 ) -> tuple [str , list [dict [str , Any ]], bool ]:
44784496 """Parses a Content or Part object into summary text and content parts.
44794497
@@ -4486,6 +4504,11 @@ async def _parse_content_object(
44864504 message index within a multi-content request) disambiguate GCS object
44874505 names: the part index alone restarts per Content, so two messages in
44884506 one request would otherwise collide at the same part ordinal.
4507+
4508+ ``offload`` controls GCS offloading. When ``False``, ``self.offloader``
4509+ is ignored: binary parts stay inline with ``"[BINARY DATA]"`` and
4510+ oversized text parts stay inline (truncated at ``max_length``) in
4511+ ``part_data["text"]`` and ``summary_text``.
44894512 """
44904513 trace_id = trace_id if trace_id is not None else self .trace_id
44914514 span_id = span_id if span_id is not None else self .span_id
@@ -4532,7 +4555,8 @@ async def _parse_content_object(
45324555 elif part .inline_data :
45334556 inline_data = part .inline_data
45344557 mime_type = inline_data .mime_type or "application/octet-stream"
4535- if self .offloader :
4558+ part_data ["mime_type" ] = mime_type
4559+ if self .offloader and offload :
45364560 ext = mimetypes .guess_extension (mime_type ) or ".bin"
45374561 path = (
45384562 f"{ datetime .now ().date ()} /{ trace_id } /{ span_id } _{ parse_uid } "
@@ -4553,7 +4577,6 @@ async def _parse_content_object(
45534577 ),
45544578 }
45554579 part_data ["object_ref" ] = object_ref
4556- part_data ["mime_type" ] = mime_type
45574580 part_data ["text" ] = "[MEDIA OFFLOADED]"
45584581 except Exception as e :
45594582 logger .warning ("Failed to offload content to GCS: %s" , e )
@@ -4577,7 +4600,11 @@ async def _parse_content_object(
45774600 self .max_length != - 1 and char_len > self .max_length
45784601 )
45794602
4580- if self .offloader and (exceeds_inline_byte_limit or exceeds_char_limit ):
4603+ if (
4604+ self .offloader
4605+ and offload
4606+ and (exceeds_inline_byte_limit or exceeds_char_limit )
4607+ ):
45814608 # Text is too big, treat as file
45824609 path = (
45834610 f"{ datetime .now ().date ()} /{ trace_id } /{ span_id } _{ parse_uid } "
@@ -4608,7 +4635,7 @@ async def _parse_content_object(
46084635 part_data ["text" ] = clean_text
46094636 summary_text .append (clean_text )
46104637 else :
4611- # Text is small or no offloader , keep inline
4638+ # Text is small or offloading is disabled , keep inline
46124639 clean_text , truncated = self ._truncate (safe_text )
46134640 if truncated :
46144641 is_truncated = True
@@ -4694,6 +4721,7 @@ async def parse(
46944721 * ,
46954722 trace_id : Optional [str ] = None ,
46964723 span_id : Optional [str ] = None ,
4724+ offload : bool = True ,
46974725 ) -> tuple [Any , list [dict [str , Any ]], bool ]:
46984726 """Parses content into JSON payload and content parts, potentially offloading to GCS.
46994727
@@ -4702,6 +4730,10 @@ async def parse(
47024730 events, so relying on the mutable instance fields lets one event's await
47034731 resume with another event's identity and overwrite its objects. The instance
47044732 fields remain only as a backward-compatible default.
4733+
4734+ ``offload`` controls whether eligible binary and oversized text parts are
4735+ uploaded via ``self.offloader``. When ``False``, parts stay inline
4736+ (``"[BINARY DATA]"`` for binary parts and truncated text for text parts).
47054737 """
47064738 trace_id = trace_id if trace_id is not None else self .trace_id
47074739 span_id = span_id if span_id is not None else self .span_id
@@ -4731,6 +4763,7 @@ def process_text(t: str) -> tuple[str, bool]:
47314763 span_id = span_id ,
47324764 parse_uid = parse_uid ,
47334765 content_ordinal = content_idx ,
4766+ offload = offload ,
47344767 )
47354768 if trunc :
47364769 is_truncated = True
@@ -4755,6 +4788,7 @@ def process_text(t: str) -> tuple[str, bool]:
47554788 span_id = span_id ,
47564789 parse_uid = parse_uid ,
47574790 content_ordinal = len (contents ),
4791+ offload = offload ,
47584792 )
47594793 if trunc :
47604794 is_truncated = True
@@ -4767,6 +4801,7 @@ def process_text(t: str) -> tuple[str, bool]:
47674801 trace_id = trace_id ,
47684802 span_id = span_id ,
47694803 parse_uid = parse_uid ,
4804+ offload = offload ,
47704805 )
47714806 return {"text_summary" : summary }, parts , trunc
47724807
@@ -6041,6 +6076,7 @@ def __init__(
60416076 # client and the executor it is not invalidated by shutdown, fork, or
60426077 # pickling, and the check is not repeated once it has passed.
60436078 self ._schema_ready = False
6079+ self ._offload_disabled_warned = False
60446080 # Workflow observation segments of the runner calls in flight, keyed by
60456081 # invocation ID: before_run_callback opens a call's segment, and
60466082 # after_run_callback or on_run_error_callback closes it. A nested runner
@@ -6053,6 +6089,13 @@ def __init__(
60536089 self ._init_pid = os .getpid ()
60546090 _LIVE_PLUGINS .add (self )
60556091
6092+ def _content_parts_persisted (self ) -> bool :
6093+ """Returns True when ``content_parts`` is persisted on the row."""
6094+ return bool (
6095+ self .config .log_multi_modal_content
6096+ and "content_parts" not in self ._denied_columns
6097+ )
6098+
60566099 def _get_bg_loop_key (self ) -> tuple [str , Any ]:
60576100 """Returns the cache key for shared background-loop states."""
60586101 creds_key = (
@@ -6819,19 +6862,35 @@ def _close() -> None:
68196862
68206863 self .offloader = None
68216864 if self .config .gcs_bucket_name :
6822- if "content_parts" in self ._denied_columns :
6865+ if not self ._content_parts_persisted () :
68236866 # GCS offload stores its object reference in the
6824- # ``content_parts`` column. With ``content_parts`` projected out,
6825- # an upload would be orphaned -- payload leaks to GCS and incurs
6826- # cost with no retained reference. Disable offload and keep
6827- # content inline (truncated) instead.
6828- logger .warning (
6829- "GCS offload disabled: payload_column_denylist drops"
6830- " 'content_parts', which holds the offloaded object reference;"
6831- " large/binary content is kept inline (truncated) instead of"
6832- " being uploaded to %s." ,
6833- self .config .gcs_bucket_name ,
6834- )
6867+ # ``content_parts`` column. When ``content_parts`` is projected
6868+ # out by ``payload_column_denylist`` or cleared to ``[]`` because
6869+ # ``log_multi_modal_content`` is False, an upload would be
6870+ # orphaned -- payload leaks to GCS, incurs cost, and large text is
6871+ # omitted from ``content`` with no retained reference. Disable
6872+ # offload so large text is kept inline (truncated) in ``content``
6873+ # and binary parts are not stored.
6874+ if not self ._offload_disabled_warned :
6875+ if "content_parts" in self ._denied_columns :
6876+ reason = (
6877+ "payload_column_denylist drops 'content_parts', which"
6878+ " holds the offloaded object reference"
6879+ )
6880+ else :
6881+ reason = (
6882+ "log_multi_modal_content is False, so 'content_parts'"
6883+ " (which holds the offloaded object reference) is not"
6884+ " persisted"
6885+ )
6886+ logger .warning (
6887+ "GCS offload disabled: %s; large text is kept inline"
6888+ " (truncated) in 'content' and binary parts are not"
6889+ " stored instead of being uploaded to %s." ,
6890+ reason ,
6891+ self .config .gcs_bucket_name ,
6892+ )
6893+ self ._offload_disabled_warned = True
68356894 else :
68366895 self .offloader = GCSOffloader (
68376896 self .project_id ,
@@ -7788,6 +7847,7 @@ def __setstate__(self, state: dict[str, Any]) -> None:
77887847 state .setdefault ("_setup_failures" , 0 )
77897848 state .setdefault ("_setup_retry_at" , 0.0 )
77907849 state .setdefault ("_schema_ready" , False )
7850+ state .setdefault ("_offload_disabled_warned" , False )
77917851 state .setdefault ("_loop_state_futures" , {})
77927852 state .setdefault ("_builder_waiters" , {})
77937853 # Migrate the legacy _visual_builder flag onto _surface. Handle an
@@ -8823,13 +8883,15 @@ async def _log_event(
88238883 logger .warning ("Parser not initialized; skipping event %s." , event_type )
88248884 return
88258885
8826- # When both payload columns are projected out, skip content parsing
8827- # entirely -- no inline summary, no parts, and (critically) no GCS offload
8828- # work for a row that retains neither payload column.
8886+ # When neither payload column will store parsed output (``content`` is
8887+ # projected out and ``content_parts`` is either projected out or disabled
8888+ # via ``log_multi_modal_content=False``), skip content parsing entirely
8889+ # -- no inline summary, no parts, and no GCS offload work.
8890+ content_parts_persisted = self ._content_parts_persisted ()
88298891 content_json : Any
88308892 content_parts : list [dict [str , Any ]]
88318893 parser_truncated : bool
8832- if { "content" , "content_parts" } <= self ._denied_columns :
8894+ if "content" in self ._denied_columns and not content_parts_persisted :
88338895 content_json , content_parts , parser_truncated = None , [], False
88348896 else :
88358897 # Pass trace/span per call: the parser instance is shared, so storing
@@ -8840,6 +8902,7 @@ async def _log_event(
88408902 raw_content ,
88418903 trace_id = trace_id or "no_trace" ,
88428904 span_id = span_id or "no_span" ,
8905+ offload = content_parts_persisted ,
88438906 )
88448907 # Normalize the parser OUTPUT to strictly JSON-native values
88458908 # inside the same boundary: a nested
@@ -8925,9 +8988,7 @@ async def _log_event(
89258988 "span_id" : span_id ,
89268989 "parent_span_id" : parent_span_id ,
89278990 "content" : content_json ,
8928- "content_parts" : (
8929- content_parts if self .config .log_multi_modal_content else []
8930- ),
8991+ "content_parts" : content_parts if content_parts_persisted else [],
89318992 "attributes" : attributes_json ,
89328993 "latency_ms" : latency_json ,
89338994 "status" : event_data .status ,
0 commit comments