From c6bbede67c49a302904be849ef694c0b276d4191 Mon Sep 17 00:00:00 2001 From: atarashansky Date: Wed, 22 Nov 2023 10:33:19 -0800 Subject: [PATCH] chore: upgrade dependencies --- .github/workflows/compatibility_tests.yml | 134 +++++++++++----------- server/common/rest.py | 10 +- server/common/utils/utils.py | 2 +- server/data_common/data_adaptor.py | 6 +- server/requirements.txt | 45 ++++---- 5 files changed, 97 insertions(+), 100 deletions(-) diff --git a/.github/workflows/compatibility_tests.yml b/.github/workflows/compatibility_tests.yml index 9d7b0dace..778fe84ae 100644 --- a/.github/workflows/compatibility_tests.yml +++ b/.github/workflows/compatibility_tests.yml @@ -2,7 +2,7 @@ name: Compatibility Tests on: schedule: - - cron: '0 8 7 * 2' + - cron: "0 8 7 * 2" push: branches: - main @@ -46,78 +46,78 @@ jobs: # add anndata pinned version test for subset of matrix configurations, # in order to reduce matrix cross-product explosion include: - - python-version: 3.8 + - python-version: 3.9 cellxgene_build: latest # TODO: dynamically use the literal version in requirements.txt, # to avoid having to update this in manually in the future # TODO: Do not bother running this if anndata latest version # matches this pinned version, to avoid a redundant test - anndata_version: '==0.7.6' + anndata_version: "==0.10.3" steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Cache env vars - run: echo "PIP_CACHE=`python -m pip cache dir`" >> $GITHUB_ENV - - name: Cache env vars (MacOS) - if: startsWith(matrix.os, 'macos') - run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV - # FIXME: Only working for Linux - - name: Python cache - uses: actions/cache@v1 - with: - path: ${{ env.PIP_CACHE }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Node cache - uses: actions/cache@v1 - with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- - - name: Brew cache (MacOS) - if: startsWith(matrix.os, 'macos') - uses: actions/cache@v1 - with: - path: ${{ env.BREW_CACHE }} - key: ${{ runner.os }}-brew- - - name: Install dependencies (Ubuntu Linux) - if: startsWith(matrix.os, 'ubuntu') - run: | - sudo apt-get update - sudo apt-get install -y libhdf5-serial-dev - - name: Install dependencies (MacOS) - if: startsWith(matrix.os, 'macos') - run: brew install hdf5 - - name: Install cellxgene from `main` branch - if: matrix.cellxgene_build == 'main' - run: | - pip install -r server/requirements-dev.txt - make pydist install-dist - - name: Install cellxgene from latest release (pypi.org) - if: matrix.cellxgene_build == 'latest' - run: | - pip install --upgrade cellxgene - # install the additional dev requirements on top of what is in the - # cellxgene pip package, which are needed for testing, but otherwise - # keep same pip pkg versions as in the cxg release - sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt - pip install -r server/requirements-dev.txt - - name: Install anndata version per matrix variable - run: pip install anndata${{ matrix.anndata_version }} - - name: Install node - run: make dev-env-client - # Run different types of test separately, to facilitate troubleshooting - - name: Unit Tests - client - run: make unit-test-client - - name: Unit Tests - server - run: make unit-test-server - - name: Smoke Tests - run: make smoke-test + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Cache env vars + run: echo "PIP_CACHE=`python -m pip cache dir`" >> $GITHUB_ENV + - name: Cache env vars (MacOS) + if: startsWith(matrix.os, 'macos') + run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV + # FIXME: Only working for Linux + - name: Python cache + uses: actions/cache@v1 + with: + path: ${{ env.PIP_CACHE }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Node cache + uses: actions/cache@v1 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + - name: Brew cache (MacOS) + if: startsWith(matrix.os, 'macos') + uses: actions/cache@v1 + with: + path: ${{ env.BREW_CACHE }} + key: ${{ runner.os }}-brew- + - name: Install dependencies (Ubuntu Linux) + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo apt-get update + sudo apt-get install -y libhdf5-serial-dev + - name: Install dependencies (MacOS) + if: startsWith(matrix.os, 'macos') + run: brew install hdf5 + - name: Install cellxgene from `main` branch + if: matrix.cellxgene_build == 'main' + run: | + pip install -r server/requirements-dev.txt + make pydist install-dist + - name: Install cellxgene from latest release (pypi.org) + if: matrix.cellxgene_build == 'latest' + run: | + pip install --upgrade cellxgene + # install the additional dev requirements on top of what is in the + # cellxgene pip package, which are needed for testing, but otherwise + # keep same pip pkg versions as in the cxg release + sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt + pip install -r server/requirements-dev.txt + - name: Install anndata version per matrix variable + run: pip install anndata${{ matrix.anndata_version }} + - name: Install node + run: make dev-env-client + # Run different types of test separately, to facilitate troubleshooting + - name: Unit Tests - client + run: make unit-test-client + - name: Unit Tests - server + run: make unit-test-server + - name: Smoke Tests + run: make smoke-test # FIXME: Fails intermittently. See https://app.zenhub.com/workspaces/single-cell-5e2a191dad828d52cc78b028/issues/chanzuckerberg/cellxgene/2415 # - name: Smoke Tests with Annotations # run: make smoke-test-annotations diff --git a/server/common/rest.py b/server/common/rest.py index e7ff4114c..fba6c0f59 100644 --- a/server/common/rest.py +++ b/server/common/rest.py @@ -6,7 +6,7 @@ import json from flask import make_response, jsonify, current_app, abort -from werkzeug.urls import url_unquote +from urllib.parse import unquote from server.common.config.client_config import get_client_config from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg @@ -64,22 +64,22 @@ def _query_parameter_to_filter(args): axis, name = key.split(":") if axis not in ("obs", "var"): raise FilterError("unknown filter axis") - name = url_unquote(name) + name = unquote(name) current = filters[axis].setdefault(name, {"name": name}) val_split = value.split(",") if len(val_split) == 1: if "min" in current or "max" in current: raise FilterError("do not mix range and value filters") - value = url_unquote(value) + value = unquote(value) values = current.setdefault("values", []) values.append(value) elif len(val_split) == 2: if len(current) > 1: raise FilterError("duplicate range specification") - min = url_unquote(val_split[0]) - max = url_unquote(val_split[1]) + min = unquote(val_split[0]) + max = unquote(val_split[1]) if min != "*": current["min"] = float(min) if max != "*": diff --git a/server/common/utils/utils.py b/server/common/utils/utils.py index 20b9adde3..f75940c0b 100644 --- a/server/common/utils/utils.py +++ b/server/common/utils/utils.py @@ -8,7 +8,7 @@ from urllib.parse import urlsplit, urljoin import numpy as np -from flask import json +import json from server.common.errors import ConfigurationError diff --git a/server/data_common/data_adaptor.py b/server/data_common/data_adaptor.py index 550028a41..d93561914 100644 --- a/server/data_common/data_adaptor.py +++ b/server/data_common/data_adaptor.py @@ -154,7 +154,7 @@ def update_parameters(self, parameters): parameters.update(self.parameters) def _index_filter_to_mask(self, filter, count): - mask = np.zeros((count,), dtype=np.bool) + mask = np.zeros((count,), dtype='bool') for i in filter: if isinstance(i, list): mask[i[0] : i[1]] = True @@ -163,7 +163,7 @@ def _index_filter_to_mask(self, filter, count): return mask def _axis_filter_to_mask(self, axis, filter, count): - mask = np.ones((count,), dtype=np.bool) + mask = np.ones((count,), dtype='bool') if "index" in filter: mask = np.logical_and(mask, self._index_filter_to_mask(filter["index"], count)) if "annotation_value" in filter: @@ -172,7 +172,7 @@ def _axis_filter_to_mask(self, axis, filter, count): return mask def _annotation_filter_to_mask(self, axis, filter, count): - mask = np.ones((count,), dtype=np.bool) + mask = np.ones((count,), dtype='bool') for v in filter: name = v["name"] if axis == Axis.VAR: diff --git a/server/requirements.txt b/server/requirements.txt index 75b05de62..5a2f432d7 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,25 +1,22 @@ -# NOTE: If you update 'anndata' min version, also update the 'anndata_version' -# matrix value in .github/workflows/compatibility_tests.yml -anndata>=0.7.6 # we need to_memory(), added in 0.7.6 -boto3>=1.12.18 -click>=7.1.2 -Flask>=1.0.2,<2.3.0 -Flask-Compress>=1.4.0 -Flask-Cors>=3.0.9 # CVE-2020-25032 -Flask-RESTful>=0.3.6 -flask-server-timing>=0.1.2 -flask-talisman>=0.7.0 -flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires migration -flatten-dict>=0.2.0 -fsspec>=0.4.4,<0.8.0 -gunicorn>=20.0.4 -h5py>=3.0.0 -numba>=0.51.2 -numpy>=1.17.5,<=1.22 -packaging>=20.0 -pandas>=1.0,!=1.1 # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446 -PyYAML>=5.4 # CVE-2020-14343 -scipy>=1.4 -requests>=2.22.0 +anndata==0.10.3 +boto3==1.29.5 +click==8.1.7 +Flask==3.0.0 +Flask-Compress==1.14 +Flask-Cors==4.0.0 +Flask-RESTful==0.3.10 +flask-server-timing==0.1.2 +flask-talisman==1.1.0 +flatbuffers==1.12 +flatten-dict==0.4.2 +fsspec==2023.10.0 +gunicorn==21.2.0 +h5py==3.10.0 +numba==0.58.1 +numpy==1.26.2 +packaging==23.2 +pandas==2.1.3 +PyYAML==6.0.1 +requests==2.31.0 s3fs==0.4.2 -# Werkzeug>=2.2.0,<3.0.0 # our version of flask doesn't support 3.0.0 +scipy==1.11.4 \ No newline at end of file