chore: bump pyarrow and pandas (#12882)
* bump pyarrow and pandas
* remove df copy
diff --git a/requirements/base.txt b/requirements/base.txt
index 2edf141..2d5f824 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -5,107 +5,301 @@
#
# pip-compile-multi
#
--e file:. # via -r requirements/base.in
-aiohttp==3.7.2 # via slackclient
-alembic==1.4.3 # via flask-migrate
-amqp==2.6.1 # via kombu
-apispec[yaml]==3.3.2 # via flask-appbuilder
-async-timeout==3.0.1 # via aiohttp
-attrs==20.2.0 # via aiohttp, jsonschema
-babel==2.8.0 # via flask-babel
-backoff==1.10.0 # via apache-superset
-billiard==3.6.3.0 # via celery
-bleach==3.2.1 # via apache-superset
-brotli==1.0.9 # via flask-compress
-cachelib==0.1.1 # via apache-superset
-celery==4.4.7 # via apache-superset
-cffi==1.14.3 # via cryptography
-chardet==3.0.4 # via aiohttp
-click==7.1.2 # via apache-superset, flask, flask-appbuilder
-colorama==0.4.4 # via apache-superset, flask-appbuilder
-contextlib2==0.6.0.post1 # via apache-superset
-convertdate==2.3.0 # via holidays
-cron-descriptor==1.2.24 # via apache-superset
-croniter==0.3.36 # via apache-superset
-cryptography==3.2.1 # via apache-superset
-decorator==4.4.2 # via retry
-defusedxml==0.6.0 # via python3-openid
-dnspython==2.0.0 # via email-validator
-email-validator==1.1.1 # via flask-appbuilder
-flask-appbuilder==3.1.1 # via apache-superset
-flask-babel==1.0.0 # via flask-appbuilder
-flask-caching==1.9.0 # via apache-superset
-flask-compress==1.8.0 # via apache-superset
-flask-jwt-extended==3.24.1 # via flask-appbuilder
-flask-login==0.4.1 # via flask-appbuilder
-flask-migrate==2.5.3 # via apache-superset
-flask-openid==1.2.5 # via flask-appbuilder
-flask-sqlalchemy==2.4.4 # via flask-appbuilder, flask-migrate
-flask-talisman==0.7.0 # via apache-superset
-flask-wtf==0.14.3 # via apache-superset, flask-appbuilder
-flask==1.1.2 # via apache-superset, flask-appbuilder, flask-babel, flask-caching, flask-compress, flask-jwt-extended, flask-login, flask-migrate, flask-openid, flask-sqlalchemy, flask-wtf
-geographiclib==1.50 # via geopy
-geopy==2.0.0 # via apache-superset
-gunicorn==20.0.4 # via apache-superset
-holidays==0.10.3 # via apache-superset
-humanize==3.1.0 # via apache-superset
-idna==2.10 # via email-validator, yarl
-importlib-metadata==2.1.1 # via -r requirements/base.in, jsonschema, kombu, markdown
-isodate==0.6.0 # via apache-superset
-itsdangerous==1.1.0 # via flask, flask-wtf
-jinja2==2.11.3 # via flask, flask-babel
-jsonschema==3.2.0 # via flask-appbuilder
-kombu==4.6.11 # via celery
-korean-lunar-calendar==0.2.1 # via holidays
-mako==1.1.3 # via alembic
-markdown==3.3.3 # via apache-superset
-markupsafe==1.1.1 # via jinja2, mako, wtforms
-marshmallow-enum==1.5.1 # via flask-appbuilder
-marshmallow-sqlalchemy==0.23.1 # via flask-appbuilder
-marshmallow==3.9.0 # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy
-msgpack==1.0.0 # via apache-superset
-multidict==5.0.0 # via aiohttp, yarl
-natsort==7.0.1 # via croniter
-numpy==1.19.4 # via pandas, pyarrow
-packaging==20.4 # via bleach
-pandas==1.1.4 # via apache-superset
-parsedatetime==2.6 # via apache-superset
-pathlib2==2.3.5 # via apache-superset
-pgsanity==0.2.9 # via apache-superset
-polyline==1.4.0 # via apache-superset
-prison==0.1.3 # via flask-appbuilder
-py==1.9.0 # via retry
-pyarrow==1.0.1 # via apache-superset
-pycparser==2.20 # via cffi
-pyjwt==1.7.1 # via apache-superset, flask-appbuilder, flask-jwt-extended
-pymeeus==0.3.7 # via convertdate
-pyparsing==2.4.7 # via apache-superset, packaging
-pyrsistent==0.16.1 # via -r requirements/base.in, jsonschema
-python-dateutil==2.8.1 # via alembic, apache-superset, croniter, flask-appbuilder, holidays, pandas
-python-dotenv==0.15.0 # via apache-superset
-python-editor==1.0.4 # via alembic
-python-geohash==0.8.5 # via apache-superset
-python3-openid==3.2.0 # via flask-openid
-pytz==2020.4 # via babel, celery, convertdate, flask-babel, pandas
-pyyaml==5.3.1 # via apache-superset, apispec
-redis==3.5.3 # via apache-superset
-retry==0.9.2 # via apache-superset
-selenium==3.141.0 # via apache-superset
-simplejson==3.17.2 # via apache-superset
-six==1.15.0 # via bleach, cryptography, flask-jwt-extended, flask-talisman, holidays, isodate, jsonschema, packaging, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json
-slackclient==2.5.0 # via apache-superset
-sqlalchemy-utils==0.36.8 # via apache-superset, flask-appbuilder
-sqlalchemy==1.3.20 # via alembic, apache-superset, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
-sqlparse==0.3.0 # via apache-superset
-typing-extensions==3.7.4.3 # via aiohttp, apache-superset, yarl
-urllib3==1.25.11 # via selenium
-vine==1.3.0 # via amqp, celery
-webencodings==0.5.1 # via bleach
-werkzeug==1.0.1 # via flask, flask-jwt-extended
-wtforms-json==0.3.3 # via apache-superset
-wtforms==2.3.3 # via flask-wtf, wtforms-json
-yarl==1.6.2 # via aiohttp
-zipp==3.4.0 # via importlib-metadata
+-e file:.
+# via -r requirements/base.in
+aiohttp==3.7.2
+# via slackclient
+alembic==1.4.3
+# via flask-migrate
+amqp==2.6.1
+# via kombu
+apispec[yaml]==3.3.2
+# via flask-appbuilder
+async-timeout==3.0.1
+# via aiohttp
+attrs==20.2.0
+# via
+# aiohttp
+# jsonschema
+babel==2.8.0
+# via flask-babel
+backoff==1.10.0
+# via apache-superset
+billiard==3.6.3.0
+# via celery
+bleach==3.2.1
+# via apache-superset
+brotli==1.0.9
+# via flask-compress
+cachelib==0.1.1
+# via apache-superset
+celery==4.4.7
+# via apache-superset
+cffi==1.14.3
+# via cryptography
+chardet==3.0.4
+# via aiohttp
+click==7.1.2
+# via
+# apache-superset
+# flask
+# flask-appbuilder
+colorama==0.4.4
+# via
+# apache-superset
+# flask-appbuilder
+contextlib2==0.6.0.post1
+# via apache-superset
+convertdate==2.3.0
+# via holidays
+cron-descriptor==1.2.24
+# via apache-superset
+croniter==0.3.36
+# via apache-superset
+cryptography==3.2.1
+# via apache-superset
+decorator==4.4.2
+# via retry
+defusedxml==0.6.0
+# via python3-openid
+dnspython==2.0.0
+# via email-validator
+email-validator==1.1.1
+# via flask-appbuilder
+flask-appbuilder==3.1.1
+# via apache-superset
+flask-babel==1.0.0
+# via flask-appbuilder
+flask-caching==1.9.0
+# via apache-superset
+flask-compress==1.8.0
+# via apache-superset
+flask-jwt-extended==3.24.1
+# via flask-appbuilder
+flask-login==0.4.1
+# via flask-appbuilder
+flask-migrate==2.5.3
+# via apache-superset
+flask-openid==1.2.5
+# via flask-appbuilder
+flask-sqlalchemy==2.4.4
+# via
+# flask-appbuilder
+# flask-migrate
+flask-talisman==0.7.0
+# via apache-superset
+flask-wtf==0.14.3
+# via
+# apache-superset
+# flask-appbuilder
+flask==1.1.2
+# via
+# apache-superset
+# flask-appbuilder
+# flask-babel
+# flask-caching
+# flask-compress
+# flask-jwt-extended
+# flask-login
+# flask-migrate
+# flask-openid
+# flask-sqlalchemy
+# flask-wtf
+geographiclib==1.50
+# via geopy
+geopy==2.0.0
+# via apache-superset
+gunicorn==20.0.4
+# via apache-superset
+holidays==0.10.3
+# via apache-superset
+humanize==3.1.0
+# via apache-superset
+idna==2.10
+# via
+# email-validator
+# yarl
+importlib-metadata==2.1.1
+# via
+# -r requirements/base.in
+# jsonschema
+# kombu
+# markdown
+isodate==0.6.0
+# via apache-superset
+itsdangerous==1.1.0
+# via
+# flask
+# flask-wtf
+jinja2==2.11.3
+# via
+# flask
+# flask-babel
+jsonschema==3.2.0
+# via flask-appbuilder
+kombu==4.6.11
+# via celery
+korean-lunar-calendar==0.2.1
+# via holidays
+mako==1.1.3
+# via alembic
+markdown==3.3.3
+# via apache-superset
+markupsafe==1.1.1
+# via
+# jinja2
+# mako
+# wtforms
+marshmallow-enum==1.5.1
+# via flask-appbuilder
+marshmallow-sqlalchemy==0.23.1
+# via flask-appbuilder
+marshmallow==3.9.0
+# via
+# flask-appbuilder
+# marshmallow-enum
+# marshmallow-sqlalchemy
+msgpack==1.0.0
+# via apache-superset
+multidict==5.0.0
+# via
+# aiohttp
+# yarl
+natsort==7.0.1
+# via croniter
+numpy==1.19.4
+# via
+# pandas
+# pyarrow
+packaging==20.4
+# via bleach
+pandas==1.2.2
+# via apache-superset
+parsedatetime==2.6
+# via apache-superset
+pathlib2==2.3.5
+# via apache-superset
+pgsanity==0.2.9
+# via apache-superset
+polyline==1.4.0
+# via apache-superset
+prison==0.1.3
+# via flask-appbuilder
+py==1.9.0
+# via retry
+pyarrow==3.0.0
+# via apache-superset
+pycparser==2.20
+# via cffi
+pyjwt==1.7.1
+# via
+# apache-superset
+# flask-appbuilder
+# flask-jwt-extended
+pymeeus==0.3.7
+# via convertdate
+pyparsing==2.4.7
+# via
+# apache-superset
+# packaging
+pyrsistent==0.16.1
+# via
+# -r requirements/base.in
+# jsonschema
+python-dateutil==2.8.1
+# via
+# alembic
+# apache-superset
+# croniter
+# flask-appbuilder
+# holidays
+# pandas
+python-dotenv==0.15.0
+# via apache-superset
+python-editor==1.0.4
+# via alembic
+python-geohash==0.8.5
+# via apache-superset
+python3-openid==3.2.0
+# via flask-openid
+pytz==2020.4
+# via
+# babel
+# celery
+# convertdate
+# flask-babel
+# pandas
+pyyaml==5.3.1
+# via
+# apache-superset
+# apispec
+redis==3.5.3
+# via apache-superset
+retry==0.9.2
+# via apache-superset
+selenium==3.141.0
+# via apache-superset
+simplejson==3.17.2
+# via apache-superset
+six==1.15.0
+# via
+# bleach
+# cryptography
+# flask-jwt-extended
+# flask-talisman
+# holidays
+# isodate
+# jsonschema
+# packaging
+# pathlib2
+# polyline
+# prison
+# pyrsistent
+# python-dateutil
+# sqlalchemy-utils
+# wtforms-json
+slackclient==2.5.0
+# via apache-superset
+sqlalchemy-utils==0.36.8
+# via
+# apache-superset
+# flask-appbuilder
+sqlalchemy==1.3.20
+# via
+# alembic
+# apache-superset
+# flask-sqlalchemy
+# marshmallow-sqlalchemy
+# sqlalchemy-utils
+sqlparse==0.3.0
+# via apache-superset
+typing-extensions==3.7.4.3
+# via
+# aiohttp
+# apache-superset
+# yarl
+urllib3==1.25.11
+# via selenium
+vine==1.3.0
+# via
+# amqp
+# celery
+webencodings==0.5.1
+# via bleach
+werkzeug==1.0.1
+# via
+# flask
+# flask-jwt-extended
+wtforms-json==0.3.3
+# via apache-superset
+wtforms==2.3.3
+# via
+# flask-wtf
+# wtforms-json
+yarl==1.6.2
+# via aiohttp
+zipp==3.4.0
+# via importlib-metadata
# The following packages are considered to be unsafe in a requirements file:
# setuptools
diff --git a/setup.py b/setup.py
index 78be764..8cba022 100644
--- a/setup.py
+++ b/setup.py
@@ -87,7 +87,7 @@
"isodate",
"markdown>=3.0",
"msgpack>=1.0.0, <1.1",
- "pandas>=1.1.2, <1.2",
+ "pandas>=1.2.2, <1.3",
"parsedatetime",
"pathlib2",
"pgsanity",
@@ -95,7 +95,7 @@
"python-dateutil",
"python-dotenv",
"python-geohash",
- "pyarrow>=1.0.1, <1.1",
+ "pyarrow>=3.0.0, <3.1",
"pyyaml>=5.1",
"PyJWT>=1.7.1, <2",
"redis",
diff --git a/superset/common/query_context.py b/superset/common/query_context.py
index 58b5238..a740af5 100644
--- a/superset/common/query_context.py
+++ b/superset/common/query_context.py
@@ -119,7 +119,7 @@
# If the datetime format is unix, the parse will use the corresponding
# parsing logic
if not df.empty:
- df = normalize_dttm_col(
+ normalize_dttm_col(
df=df,
timestamp_format=timestamp_format,
offset=self.datasource.offset,
diff --git a/superset/utils/core.py b/superset/utils/core.py
index 4efad98..4c48f73 100644
--- a/superset/utils/core.py
+++ b/superset/utils/core.py
@@ -1604,10 +1604,9 @@
timestamp_format: Optional[str],
offset: int,
time_shift: Optional[timedelta],
-) -> pd.DataFrame:
+) -> None:
if DTTM_ALIAS not in df.columns:
- return df
- df = df.copy()
+ return
if timestamp_format in ("epoch_s", "epoch_ms"):
dttm_col = df[DTTM_ALIAS]
if is_numeric_dtype(dttm_col):
@@ -1627,4 +1626,3 @@
df[DTTM_ALIAS] += timedelta(hours=offset)
if time_shift is not None:
df[DTTM_ALIAS] += time_shift
- return df
diff --git a/superset/viz.py b/superset/viz.py
index b403899..1183418 100644
--- a/superset/viz.py
+++ b/superset/viz.py
@@ -284,7 +284,7 @@
# If the datetime format is unix, the parse will use the corresponding
# parsing logic.
if not df.empty:
- df = utils.normalize_dttm_col(
+ utils.normalize_dttm_col(
df=df,
timestamp_format=timestamp_format,
offset=self.datasource.offset,
diff --git a/tests/utils_tests.py b/tests/utils_tests.py
index 9fed172..d810c40 100644
--- a/tests/utils_tests.py
+++ b/tests/utils_tests.py
@@ -23,7 +23,7 @@
import json
import os
import re
-from typing import Any, Tuple, List
+from typing import Any, Tuple, List, Optional
from unittest.mock import Mock, patch
from tests.fixtures.birth_names_dashboard import load_birth_names_dashboard_with_slices
@@ -1135,28 +1135,38 @@
assert extract_dataframe_dtypes(df) == [col[1] for col in cols]
def test_normalize_dttm_col(self):
+ def normalize_col(
+ df: pd.DataFrame,
+ timestamp_format: Optional[str],
+ offset: int,
+ time_shift: Optional[timedelta],
+ ) -> pd.DataFrame:
+ df = df.copy()
+ normalize_dttm_col(df, timestamp_format, offset, time_shift)
+ return df
+
ts = pd.Timestamp(2021, 2, 15, 19, 0, 0, 0)
df = pd.DataFrame([{"__timestamp": ts, "a": 1}])
# test regular (non-numeric) format
- assert normalize_dttm_col(df, None, 0, None)[DTTM_ALIAS][0] == ts
- assert normalize_dttm_col(df, "epoch_ms", 0, None)[DTTM_ALIAS][0] == ts
- assert normalize_dttm_col(df, "epoch_s", 0, None)[DTTM_ALIAS][0] == ts
+ assert normalize_col(df, None, 0, None)[DTTM_ALIAS][0] == ts
+ assert normalize_col(df, "epoch_ms", 0, None)[DTTM_ALIAS][0] == ts
+ assert normalize_col(df, "epoch_s", 0, None)[DTTM_ALIAS][0] == ts
# test offset
- assert normalize_dttm_col(df, None, 1, None)[DTTM_ALIAS][0] == pd.Timestamp(
+ assert normalize_col(df, None, 1, None)[DTTM_ALIAS][0] == pd.Timestamp(
2021, 2, 15, 20, 0, 0, 0
)
# test offset and timedelta
- assert normalize_dttm_col(df, None, 1, timedelta(minutes=30))[DTTM_ALIAS][
+ assert normalize_col(df, None, 1, timedelta(minutes=30))[DTTM_ALIAS][
0
] == pd.Timestamp(2021, 2, 15, 20, 30, 0, 0)
# test numeric epoch_s format
df = pd.DataFrame([{"__timestamp": ts.timestamp(), "a": 1}])
- assert normalize_dttm_col(df, "epoch_s", 0, None)[DTTM_ALIAS][0] == ts
+ assert normalize_col(df, "epoch_s", 0, None)[DTTM_ALIAS][0] == ts
# test numeric epoch_ms format
df = pd.DataFrame([{"__timestamp": ts.timestamp() * 1000, "a": 1}])
- assert normalize_dttm_col(df, "epoch_ms", 0, None)[DTTM_ALIAS][0] == ts
+ assert normalize_col(df, "epoch_ms", 0, None)[DTTM_ALIAS][0] == ts
