Merge remote-tracking branch 'upstream/main' into issue_61863

Author: Khemkaran

.github/workflows/unit-tests.yml

@@ -30,7 +30,7 @@ jobs:
         env_file: [actions-310.yaml, actions-311.yaml, actions-312.yaml, actions-313.yaml]
         # Prevent the include jobs from overriding other jobs
         pattern: [""]
-        pandas_future_infer_string: ["0"]
+        pandas_future_infer_string: ["1"]
         include:
           - name: "Downstream Compat"
             env_file: actions-311-downstream_compat.yaml
@@ -45,6 +45,10 @@ jobs:
             env_file: actions-313-freethreading.yaml
             pattern: "not slow and not network and not single_cpu"
             platform: ubuntu-24.04
+          - name: "Without PyArrow"
+            env_file: actions-312.yaml
+            pattern: "not slow and not network and not single_cpu"
+            platform: ubuntu-24.04
           - name: "Locale: it_IT"
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -67,13 +71,9 @@ jobs:
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "zh_CN"
             platform: ubuntu-24.04
-          - name: "Future infer strings"
+          - name: "Past no infer strings"
             env_file: actions-312.yaml
-            pandas_future_infer_string: "1"
-            platform: ubuntu-24.04
-          - name: "Future infer strings (without pyarrow)"
-            env_file: actions-311.yaml
-            pandas_future_infer_string: "1"
+            pandas_future_infer_string: "0"
             platform: ubuntu-24.04
           - name: "Numpy Dev"
             env_file: actions-311-numpydev.yaml
@@ -83,7 +83,6 @@ jobs:
           - name: "Pyarrow Nightly"
             env_file: actions-311-pyarrownightly.yaml
             pattern: "not slow and not network and not single_cpu"
-            pandas_future_infer_string: "1"
             platform: ubuntu-24.04
       fail-fast: false
     name: ${{ matrix.name || format('{0} {1}', matrix.platform, matrix.env_file) }}
@@ -92,13 +91,13 @@ jobs:
       LANG: ${{ matrix.lang || 'C.UTF-8' }}
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_CI: '1'
-      PANDAS_FUTURE_INFER_STRING: ${{ matrix.pandas_future_infer_string || '0' }}
+      PANDAS_FUTURE_INFER_STRING: ${{ matrix.pandas_future_infer_string || '1' }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
       PYTEST_WORKERS: 'auto'
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
       # Clipboard tests
       QT_QPA_PLATFORM: offscreen
-      REMOVE_PYARROW: ${{ matrix.name == 'Future infer strings (without pyarrow)' && '1' || '0' }}
+      REMOVE_PYARROW: ${{ matrix.name == 'Without PyArrow' && '1' || '0' }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_future_infer_string }}-${{ matrix.platform }}

README.md

@@ -115,7 +115,7 @@ details, see the commit logs at https://github.com/pandas-dev/pandas.
 ## Dependencies
 - [NumPy - Adds support for large, multi-dimensional arrays, matrices and high-level mathematical functions to operate on these arrays](https://www.numpy.org)
 - [python-dateutil - Provides powerful extensions to the standard datetime module](https://dateutil.readthedocs.io/en/stable/index.html)
-- [pytz - Brings the Olson tz database into Python which allows accurate and cross platform timezone calculations](https://github.com/stub42/pytz)
+- [tzdata - Provides an IANA time zone database](https://tzdata.readthedocs.io/en/latest/)
 
 See the [full installation instructions](https://pandas.pydata.org/pandas-docs/stable/install.html#dependencies) for minimum supported versions of required, recommended and optional dependencies.
 

ci/code_checks.sh

@@ -58,7 +58,9 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
 
     MSG='Python and Cython Doctests' ; echo "$MSG"
     python -c 'import pandas as pd; pd.test(run_doctests=True)'
-    RET=$(($RET + $?)) ; echo "$MSG" "DONE"
+    # TEMP don't let doctests fail the build until all string dtype changes are fixed
+    # RET=$(($RET + $?)) ; echo "$MSG" "DONE"
+    echo "$MSG" "DONE"
 
 fi
 
@@ -72,6 +74,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Series.dt PR01" `# Accessors are implemented as classes, but we do not document the Parameters section` \
         -i "pandas.Period.freq GL08" \
         -i "pandas.Period.ordinal GL08" \
+        -i "pandas.errors.IncompatibleFrequency SA01,SS06,EX01" \
         -i "pandas.core.groupby.DataFrameGroupBy.plot PR02" \
         -i "pandas.core.groupby.SeriesGroupBy.plot PR02" \
         -i "pandas.core.resample.Resampler.quantile PR01,PR07" \

ci/deps/actions-311-downstream_compat.yaml

@@ -50,8 +50,7 @@ dependencies:
   - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2023.12.2
-  # TEMP upper pin for scipy (https://github.com/statsmodels/statsmodels/issues/9584)
-  - scipy>=1.12.0,<1.16
+  - scipy>=1.12.0
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0
   - xarray>=2024.1.1

doc/source/development/maintaining.rst

@@ -388,8 +388,11 @@ Pre-release
 
 3. Make sure the CI is green for the last commit of the branch being released.
 
-4. If not a release candidate, make sure all backporting pull requests to the branch
-   being released are merged.
+4. If not a release candidate, make sure all backporting pull requests to the
+   branch being released are merged, and no merged pull requests are missing a
+   backport (check the
+   ["Still Needs Manual Backport"](https://github.com/pandas-dev/pandas/labels/Still%20Needs%20Manual%20Backport)
+   label for this).
 
 5. Create a new issue and milestone for the version after the one being released.
    If the release was a release candidate, we would usually want to create issues and
@@ -435,6 +438,9 @@ which will be triggered when the tag is pushed.
 
     scripts/download_wheels.sh <VERSION>
 
+   ATTENTION: this is currently not downloading *all* wheels, and you have to
+   manually download the remainings wheels and sdist!
+
 4. Create a `new GitHub release <https://github.com/pandas-dev/pandas/releases/new>`_:
 
    - Tag: ``<version>``
@@ -462,15 +468,22 @@ Post-Release
 ````````````
 
 1. Update symlinks to stable documentation by logging in to our web server, and
-   editing ``/var/www/html/pandas-docs/stable`` to point to ``version/<latest-version>``
-   for major and minor releases, or ``version/<minor>`` to ``version/<patch>`` for
+   editing ``/var/www/html/pandas-docs/stable`` to point to ``version/<X.Y>``
+   for major and minor releases, or ``version/<X.Y>`` to ``version/<patch>`` for
    patch releases. The exact instructions are (replace the example version numbers by
    the appropriate ones for the version you are releasing):
 
    - Log in to the server and use the correct user.
    - ``cd /var/www/html/pandas-docs/``
-   - ``ln -sfn version/2.1 stable`` (for a major or minor release)
-   - ``ln -sfn version/2.0.3 version/2.0`` (for a patch release)
+   - For a major or minor release (assuming the ``/version/2.1.0/`` docs have been uploaded to the server):
+
+     -  Create a new X.Y symlink to X.Y.Z: ``cd version; ln -sfn 2.1.0 2.1``
+     -  Update stable symlink to point to X.Y: ``ln -sfn version/2.1 stable``
+
+   - For a patch release (assuming the ``/version/2.1.3/`` docs have been uploaded to the server):
+
+     - Update the X.Y symlink to the new X.Y.Z patch version: ``cd version; ln -sfn 2.1.3 2.1``
+     - (the stable symlink should already be pointing to the correct X.Y version)
 
 2. If releasing a major or minor release, open a PR in our source code to update
    ``web/pandas/versions.json``, to have the desired versions in the documentation

doc/source/reference/testing.rst

@@ -36,6 +36,7 @@ Exceptions and warnings
    errors.DuplicateLabelError
    errors.EmptyDataError
    errors.IncompatibilityWarning
+   errors.IncompatibleFrequency
    errors.IndexingError
    errors.InvalidColumnName
    errors.InvalidComparison

doc/source/user_guide/basics.rst

@@ -590,7 +590,7 @@ arguments. The special value ``all`` can also be used:
 
 .. ipython:: python
 
-    frame.describe(include=["object"])
+    frame.describe(include=["str"])
     frame.describe(include=["number"])
     frame.describe(include="all")
 

doc/source/user_guide/io.rst

@@ -5228,33 +5228,32 @@ languages easy. Parquet can use a variety of compression techniques to shrink th
 while still maintaining good read performance.
 
 Parquet is designed to faithfully serialize and de-serialize ``DataFrame`` s, supporting all of the pandas
-dtypes, including extension dtypes such as datetime with tz.
+dtypes, including extension dtypes such as datetime with timezone.
 
 Several caveats.
 
 * Duplicate column names and non-string columns names are not supported.
-* The ``pyarrow`` engine always writes the index to the output, but ``fastparquet`` only writes non-default
-  indexes. This extra column can cause problems for non-pandas consumers that are not expecting it. You can
-  force including or omitting indexes with the ``index`` argument, regardless of the underlying engine.
+* The DataFrame index is written as separate column(s) when it is a non-default range index.
+  This extra column can cause problems for non-pandas consumers that are not expecting it. You can
+  force including or omitting indexes with the ``index`` argument.
 * Index level names, if specified, must be strings.
 * In the ``pyarrow`` engine, categorical dtypes for non-string types can be serialized to parquet, but will de-serialize as their primitive dtype.
-* The ``pyarrow`` engine preserves the ``ordered`` flag of categorical dtypes with string types. ``fastparquet`` does not preserve the ``ordered`` flag.
-* Non supported types include ``Interval`` and actual Python object types. These will raise a helpful error message
-  on an attempt at serialization. ``Period`` type is supported with pyarrow >= 0.16.0.
+* The ``pyarrow`` engine supports the ``Period`` and ``Interval`` dtypes. ``fastparquet`` does not support those.
+* Non supported types include actual Python object types. These will raise a helpful error message
+  on an attempt at serialization.
 * The ``pyarrow`` engine preserves extension data types such as the nullable integer and string data
-  type (requiring pyarrow >= 0.16.0, and requiring the extension type to implement the needed protocols,
+  type (this can also work for external extension types, requiring the extension type to implement the needed protocols,
   see the :ref:`extension types documentation <extending.extension.arrow>`).
 
 You can specify an ``engine`` to direct the serialization. This can be one of ``pyarrow``, or ``fastparquet``, or ``auto``.
 If the engine is NOT specified, then the ``pd.options.io.parquet.engine`` option is checked; if this is also ``auto``,
-then ``pyarrow`` is tried, and falling back to ``fastparquet``.
+then ``pyarrow`` is used when installed, and falling back to ``fastparquet``.
 
 See the documentation for `pyarrow <https://arrow.apache.org/docs/python/>`__ and `fastparquet <https://fastparquet.readthedocs.io/en/latest/>`__.
 
 .. note::
 
-   These engines are very similar and should read/write nearly identical parquet format files.
-   ``pyarrow>=8.0.0`` supports timedelta data, ``fastparquet>=0.1.4`` supports timezone aware datetimes.
+   These engines are very similar and should read/write nearly identical parquet format files for most cases.
    These libraries differ by having different underlying dependencies (``fastparquet`` by using ``numba``, while ``pyarrow`` uses a c-library).
 
 .. ipython:: python
@@ -5280,24 +5279,21 @@ Write to a parquet file.
 
 .. ipython:: python
 
-   df.to_parquet("example_pa.parquet", engine="pyarrow")
-   df.to_parquet("example_fp.parquet", engine="fastparquet")
+   # specify engine="pyarrow" or engine="fastparquet" to use a specific engine
+   df.to_parquet("example.parquet")
 
 Read from a parquet file.
 
 .. ipython:: python
 
-   result = pd.read_parquet("example_fp.parquet", engine="fastparquet")
-   result = pd.read_parquet("example_pa.parquet", engine="pyarrow")
-
+   result = pd.read_parquet("example.parquet")
    result.dtypes
 
 By setting the ``dtype_backend`` argument you can control the default dtypes used for the resulting DataFrame.
 
 .. ipython:: python
 
-   result = pd.read_parquet("example_pa.parquet", engine="pyarrow", dtype_backend="pyarrow")
-
+   result = pd.read_parquet("example.parquet", dtype_backend="pyarrow")
    result.dtypes
 
 .. note::
@@ -5309,41 +5305,36 @@ Read only certain columns of a parquet file.
 
 .. ipython:: python
 
-   result = pd.read_parquet(
-       "example_fp.parquet",
-       engine="fastparquet",
-       columns=["a", "b"],
-   )
-   result = pd.read_parquet(
-       "example_pa.parquet",
-       engine="pyarrow",
-       columns=["a", "b"],
-   )
+   result = pd.read_parquet("example.parquet", columns=["a", "b"])
    result.dtypes
 
 
 .. ipython:: python
    :suppress:
 
-   os.remove("example_pa.parquet")
-   os.remove("example_fp.parquet")
+   os.remove("example.parquet")
 
 
 Handling indexes
 ''''''''''''''''
 
 Serializing a ``DataFrame`` to parquet may include the implicit index as one or
-more columns in the output file. Thus, this code:
+more columns in the output file. For example, this code:
 
 .. ipython:: python
 
-    df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
+    df = pd.DataFrame({"a": [1, 2], "b": [3, 4]}, index=[1, 2])
     df.to_parquet("test.parquet", engine="pyarrow")
 
-creates a parquet file with *three* columns if you use ``pyarrow`` for serialization:
-``a``, ``b``, and ``__index_level_0__``. If you're using ``fastparquet``, the
-index `may or may not <https://fastparquet.readthedocs.io/en/latest/api.html#fastparquet.write>`_
-be written to the file.
+creates a parquet file with *three* columns (``a``, ``b``, and
+``__index_level_0__`` when using the ``pyarrow`` engine, or ``index``, ``a``,
+and ``b`` when using the ``fastparquet`` engine) because the index in this case
+is not a default range index. In general, the index *may or may not* be written
+to the file (see the
+`preserve_index keyword for pyarrow <https://arrow.apache.org/docs/python/pandas.html#handling-pandas-indexes>`__
+or the
+`write_index keyword for fastparquet <https://fastparquet.readthedocs.io/en/latest/api.html#fastparquet.write>`__
+to check the default behaviour).
 
 This unexpected extra column causes some databases like Amazon Redshift to reject
 the file, because that column doesn't exist in the target table.

doc/source/user_guide/migration-3-strings.rst

@@ -118,12 +118,17 @@ through the ``str`` accessor will work the same:
 Overview of behavior differences and how to address them
 ---------------------------------------------------------
 
-The dtype is no longer object dtype
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The dtype is no longer a numpy "object" dtype
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 When inferring or reading string data, the data type of the resulting DataFrame
 column or Series will silently start being the new ``"str"`` dtype instead of
-``"object"`` dtype, and this can have some impact on your code.
+the numpy ``"object"`` dtype, and this can have some impact on your code.
+
+The new string dtype is a pandas data type ("extension dtype"), and no longer a
+numpy ``np.dtype`` instance. Therefore, passing the dtype of a string column to
+numpy functions will no longer work (e.g. passing it to a ``dtype=`` argument
+of a numpy function, or using ``np.issubdtype`` to check the dtype).
 
 Checking the dtype
 ^^^^^^^^^^^^^^^^^^

doc/source/whatsnew/index.rst

@@ -24,6 +24,7 @@ Version 2.3
 .. toctree::
    :maxdepth: 2
 
+   v2.3.2
    v2.3.1
    v2.3.0