Merge branch 'main' into shuowei-fix-compiler-syntax-guards

Author: shuoweil

.librarian/state.yaml

@@ -1,7 +1,7 @@
 image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator@sha256:160860d189ff1c2f7515638478823712fa5b243e27ccc33a2728669fa1e2ed0c
 libraries:
   - id: bigframes
-    version: 2.37.0
+    version: 2.38.0
     last_generated_commit: ""
     apis: []
     source_roots:

CHANGELOG.md

@@ -4,6 +4,29 @@
 
 [1]: https://pypi.org/project/bigframes/#history
 
+## [2.38.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.37.0...v2.38.0) (2026-03-16)
+
+
+### Documentation
+
+* add notebooks to user guide page (#2505) ([5cf37888bc0b4b1b0993dadd1e0fe5ee08341ef4](https://github.com/googleapis/python-bigquery-dataframes/commit/5cf37888bc0b4b1b0993dadd1e0fe5ee08341ef4))
+* Fix typo in ExperimentOptions class docstring (#2498) ([077cb2ebe515fc5e07bcbb5dc663edd28d3eaf00](https://github.com/googleapis/python-bigquery-dataframes/commit/077cb2ebe515fc5e07bcbb5dc663edd28d3eaf00))
+
+
+### Features
+
+* add `df.bigquery` pandas accessor (#2513) ([91b6c245521218bb78b543885e1b9424278ce2ab](https://github.com/googleapis/python-bigquery-dataframes/commit/91b6c245521218bb78b543885e1b9424278ce2ab))
+* use EUC for AI IF, CLASSIFY, and SCORE when connection is not provided (#2507) ([fe94910abff28e244dd79e1540a6c2184a12eb44](https://github.com/googleapis/python-bigquery-dataframes/commit/fe94910abff28e244dd79e1540a6c2184a12eb44))
+* Add `bigframes.bigquery.rand()` function (#2501) ([5c43efb745118f506ecc30196da68e9d6f4346dc](https://github.com/googleapis/python-bigquery-dataframes/commit/5c43efb745118f506ecc30196da68e9d6f4346dc))
+* add bigquery.ml.get_insights function (#2493) ([d29a60953ac989bb2c95e6eec3010620ac776a3c](https://github.com/googleapis/python-bigquery-dataframes/commit/d29a60953ac989bb2c95e6eec3010620ac776a3c))
+* Add str, dt accessors to pd.col Expression objects (#2488) ([ce5de57019449ca77d308946df72f04289343b51](https://github.com/googleapis/python-bigquery-dataframes/commit/ce5de57019449ca77d308946df72f04289343b51))
+
+
+### Bug Fixes
+
+* handle unsupported types and empty results in describe (#2506) ([2326ad6aec15c20a66756eff093b50be484b3ba8](https://github.com/googleapis/python-bigquery-dataframes/commit/2326ad6aec15c20a66756eff093b50be484b3ba8))
+* no longer automatically use anywidget in the `%%bqsql` magics (#2504) ([43353e2bc9ffbc38b7383c24ecaac80d3b8bab32](https://github.com/googleapis/python-bigquery-dataframes/commit/43353e2bc9ffbc38b7383c24ecaac80d3b8bab32))
+
 ## [2.37.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.36.0...v2.37.0) (2026-03-03)
 
 

bigframes/bigquery/__init__.py

@@ -12,9 +12,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""This module integrates BigQuery built-in functions for use with DataFrame objects,
-such as array functions:
-https://cloud.google.com/bigquery/docs/reference/standard-sql/array_functions. """
+"""
+Access BigQuery-specific operations and namespaces within BigQuery DataFrames.
+
+This module provides specialized functions and sub-modules that expose BigQuery's
+advanced capabilities to DataFrames and Series. It acts as a bridge between the
+pandas-compatible API and the full power of BigQuery SQL.
+
+Key sub-modules include:
+
+* :mod:`bigframes.bigquery.ai`: Generative and predictive AI functions (Gemini, BQML).
+* :mod:`bigframes.bigquery.ml`: Direct access to BigQuery ML model operations.
+* :mod:`bigframes.bigquery.obj`: Support for BigQuery object tables.
+
+This module also provides direct access to optimized BigQuery functions for:
+
+* **JSON Processing:** High-performance functions like ``json_extract``, ``json_value``,
+  and ``parse_json`` for handling semi-structured data.
+* **Geospatial Analysis:** Comprehensive geographic functions such as ``st_area``,
+  ``st_distance``, and ``st_centroid`` (``ST_`` prefixed functions).
+* **Array Operations:** Tools for working with BigQuery arrays, including ``array_agg``
+  and ``array_length``.
+* **Vector Search:** Integration with BigQuery's vector search and indexing
+  capabilities for high-dimensional data.
+* **Custom SQL:** The ``sql_scalar`` function allows embedding raw SQL snippets for
+  advanced operations not yet directly mapped in the API.
+
+By using these functions, you can leverage BigQuery's high-performance engine for
+domain-specific tasks while maintaining a Python-centric development experience.
+
+For the full list of BigQuery standard SQL functions, see:
+https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-reference
+"""
 
 import sys
 

bigframes/bigquery/_operations/ai.py

@@ -880,6 +880,7 @@ def forecast(
     id_cols: Iterable[str] | None = None,
     horizon: int = 10,
     confidence_level: float = 0.95,
+    output_historical_time_series: bool = False,
     context_window: int | None = None,
 ) -> dataframe.DataFrame:
     """
@@ -914,6 +915,15 @@ def forecast(
         confidence_level (float, default 0.95):
             A FLOAT64 value that specifies the percentage of the future values that fall in the prediction interval.
             The default value is 0.95. The valid input range is [0, 1).
+        output_historical_time_series (bool, default False):
+            A BOOL value that determines whether the input data is returned
+            along with the forecasted data. Set this argument to TRUE to return
+            input data. The default value is FALSE.
+
+            Returning the input data along with the forecasted data lets you
+            compare the historical value of the data column with the forecasted
+            value of the data column, or chart the change in the data column
+            values over time.
         context_window (int, optional):
             An int value that specifies the context window length used by BigQuery ML's built-in TimesFM model.
             The context window length determines how many of the most recent data points from the input time series are use by the model.
@@ -945,6 +955,7 @@ def forecast(
         "timestamp_col": timestamp_col,
         "model": model,
         "horizon": horizon,
+        "output_historical_time_series": output_historical_time_series,
         "confidence_level": confidence_level,
     }
     if id_cols:

bigframes/bigquery/_operations/io.py

@@ -19,8 +19,8 @@
 import pandas as pd
 
 from bigframes.bigquery._operations.table import _get_table_metadata
+import bigframes.core.compile.sqlglot.sql as sql
 import bigframes.core.logging.log_adapter as log_adapter
-import bigframes.core.sql.io
 import bigframes.session
 
 
@@ -73,7 +73,7 @@ def load_data(
     """
     import bigframes.pandas as bpd
 
-    sql = bigframes.core.sql.io.load_data_ddl(
+    load_data_expr = sql.load_data(
         table_name=table_name,
         write_disposition=write_disposition,
         columns=columns,
@@ -84,11 +84,12 @@ def load_data(
         with_partition_columns=with_partition_columns,
         connection_name=connection_name,
     )
+    sql_text = sql.to_sql(load_data_expr)
 
     if session is None:
-        bpd.read_gbq_query(sql)
+        bpd.read_gbq_query(sql_text)
         session = bpd.get_global_session()
     else:
-        session.read_gbq_query(sql)
+        session.read_gbq_query(sql_text)
 
     return _get_table_metadata(bqclient=session.bqclient, table_name=table_name)

bigframes/bigquery/_operations/table.py

@@ -19,8 +19,8 @@
 import google.cloud.bigquery
 import pandas as pd
 
+import bigframes.core.compile.sqlglot.sql as sg_sql
 import bigframes.core.logging.log_adapter as log_adapter
-import bigframes.core.sql.table
 import bigframes.session
 
 
@@ -80,14 +80,16 @@ def create_external_table(
     """
     import bigframes.pandas as bpd
 
-    sql = bigframes.core.sql.table.create_external_table_ddl(
-        table_name=table_name,
-        replace=replace,
-        if_not_exists=if_not_exists,
-        columns=columns,
-        partition_columns=partition_columns,
-        connection_name=connection_name,
-        options=options,
+    sql = sg_sql.to_sql(
+        sg_sql.create_external_table(
+            table_name=table_name,
+            replace=replace,
+            if_not_exists=if_not_exists,
+            columns=columns,
+            partition_columns=partition_columns,
+            connection_name=connection_name,
+            options=options,
+        )
     )
 
     if session is None:

bigframes/bigquery/ai.py

@@ -12,9 +12,49 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""This module integrates BigQuery built-in AI functions for use with Series/DataFrame objects,
-such as AI.GENERATE_BOOL:
-https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-ai-generate-bool"""
+"""
+Integrate BigQuery built-in AI functions into your BigQuery DataFrames workflow.
+
+The ``bigframes.bigquery.ai`` module provides a Pythonic interface to leverage BigQuery ML's
+generative AI and predictive functions directly on BigQuery DataFrames and Series objects.
+These functions enable you to perform advanced AI tasks at scale without moving data
+out of BigQuery.
+
+Key capabilities include:
+
+* **Generative AI:** Use :func:`bigframes.bigquery.ai.generate` (Gemini) to
+  perform text analysis, translation, or
+  content generation. Specialized versions like
+  :func:`~bigframes.bigquery.ai.generate_bool`,
+  :func:`~bigframes.bigquery.ai.generate_int`, and
+  :func:`~bigframes.bigquery.ai.generate_double` are available for structured
+  outputs.
+* **Embeddings:** Generate vector embeddings for text using
+  :func:`~bigframes.bigquery.ai.generate_embedding`, which are essential for
+  semantic search and retrieval-augmented generation (RAG) workflows.
+* **Classification and Scoring:** Apply machine learning models to your data for
+  predictive tasks with :func:`~bigframes.bigquery.ai.classify` and
+  :func:`~bigframes.bigquery.ai.score`.
+* **Forecasting:** Predict future values in time-series data using
+  :func:`~bigframes.bigquery.ai.forecast`.
+
+**Example usage:**
+
+    >>> import bigframes.pandas as bpd
+    >>> import bigframes.bigquery as bbq
+
+    >>> df = bpd.DataFrame({
+    ...     "text_input": [
+    ...         "Is this a positive review? The food was terrible.",
+    ...     ],
+    ... })  # doctest: +SKIP
+
+    >>> # Assuming a Gemini model has been created in BigQuery as 'my_gemini_model'
+    >>> result = bq.ai.generate_text("my_gemini_model", df["text_input"])  # doctest: +SKIP
+
+For more information on the underlying BigQuery ML syntax, see:
+https://cloud.google.com/bigquery/docs/reference/standard-sql/bigqueryml-syntax-ai-generate-bool
+"""
 
 from bigframes.bigquery._operations.ai import (
     classify,

bigframes/core/compile/sqlglot/expressions/datetime_ops.py

@@ -371,7 +371,12 @@ def _(expr: TypedExpr, op: ops.ToDatetimeOp) -> sge.Expression:
         )
         return sge.Cast(this=result, to="DATETIME")
 
-    if expr.dtype in (dtypes.STRING_DTYPE, dtypes.TIMESTAMP_DTYPE):
+    if expr.dtype in (
+        dtypes.STRING_DTYPE,
+        dtypes.TIMESTAMP_DTYPE,
+        dtypes.DATETIME_DTYPE,
+        dtypes.DATE_DTYPE,
+    ):
         return sge.TryCast(this=expr.expr, to="DATETIME")
 
     value = expr.expr
@@ -396,7 +401,12 @@ def _(expr: TypedExpr, op: ops.ToTimestampOp) -> sge.Expression:
             "PARSE_TIMESTAMP", sge.convert(op.format), expr.expr, sge.convert("UTC")
         )
 
-    if expr.dtype in (dtypes.STRING_DTYPE, dtypes.DATETIME_DTYPE):
+    if expr.dtype in (
+        dtypes.STRING_DTYPE,
+        dtypes.DATETIME_DTYPE,
+        dtypes.TIMESTAMP_DTYPE,
+        dtypes.DATE_DTYPE,
+    ):
         return sge.func("TIMESTAMP", expr.expr)
 
     value = expr.expr

bigframes/core/compile/sqlglot/sql/__init__.py

@@ -22,6 +22,7 @@
     table,
     to_sql,
 )
+from bigframes.core.compile.sqlglot.sql.ddl import create_external_table, load_data
 from bigframes.core.compile.sqlglot.sql.dml import insert, replace
 
 __all__ = [
@@ -33,6 +34,9 @@
     "literal",
     "table",
     "to_sql",
+    # From ddl.py
+    "create_external_table",
+    "load_data",
     # From dml.py
     "insert",
     "replace",