chore: fixes to run on python3.8 (#2025)

* chore: fixes to run on python3.8 * fix: remove duplicated method. * fix: pin numpy and pandas wheels. * fix: misaka versions * fix: use new method. * fix: remove print. * fix: remove comment.
Scifabric · Mar 6, 2021 · 8daeccb · 8daeccb
1 parent d1ad47b
commit 8daeccb
Show file tree

Hide file tree

Showing 15 changed files with 289 additions and 285 deletions.
diff --git a/circle.yml b/circle.yml
@@ -3,7 +3,7 @@ jobs:
   build:
     working_directory: ~/circulate
     docker:
-      - image: scifabric/python3.6-ldap
+      - image: scifabric/python3.8-ldap
         environment:
           FLASK_CONFIG: testing
           TEST_DATABASE_URL: postgresql://rtester:rtester@localhost/pybossa_test?sslmode=disable
@@ -24,7 +24,7 @@ jobs:
           command: |
               git submodule sync --recursive && git submodule update --recursive --init
       - run:
-          name: Install Python deps 
+          name: Install Python deps
           command: |
             pip install -r requirements.txt
       - run:

diff --git a/pybossa/cache/project_stats.py b/pybossa/cache/project_stats.py
@@ -16,18 +16,15 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with PYBOSSA.  If not, see <http://www.gnu.org/licenses/>.
 """Cache module for project stats."""
-from flask import current_app
 from sqlalchemy.sql import text
 from pybossa.core import db
 from pybossa.cache import memoize, ONE_DAY, FIVE_MINUTES
 import pybossa.cache.projects as cached_projects
 from pybossa.model.project_stats import ProjectStats
-from flask_babel import gettext
 
 import operator
 import time
 import datetime
-import os
 
 
 session = db.slave_session
@@ -396,13 +393,14 @@ def stats_hours(project_id, period='2 week'):
 @memoize(timeout=ONE_DAY)
 def stats_format_dates(project_id, dates, dates_anon, dates_auth):
     """Format dates stats into a JSON format."""
-    dayNewStats = dict(label=gettext("Anon + Auth"), values=[])
-    dayCompletedTasks = dict(label=gettext("Completed Tasks"),
+    dayNewStats = dict(label="Anon + Auth", values=[])
+    dayCompletedTasks = dict(label="Completed Tasks",
                              disabled="True", values=[])
-    dayNewAnonStats = dict(label=gettext("Anonymous"), values=[])
-    dayNewAuthStats = dict(label=gettext("Authenticated"), values=[])
+    dayNewAnonStats = dict(label="Anonymous", values=[])
+    dayNewAuthStats = dict(label="Authenticated", values=[])
 
-    answer_dates = sorted(list(set(list(dates_anon.keys()) + list(dates_auth.keys()))))
+    answer_dates = sorted(
+        list(set(list(dates_anon.keys()) + list(dates_auth.keys()))))
     total = 0
 
     for d in sorted(dates.keys()):
@@ -435,10 +433,10 @@ def stats_format_dates(project_id, dates, dates_anon, dates_auth):
 def stats_format_hours(project_id, hours, hours_anon, hours_auth,
                        max_hours, max_hours_anon, max_hours_auth):
     """Format hours stats into a JSON format."""
-    hourNewStats = dict(label=gettext("Anon + Auth"),
+    hourNewStats = dict(label="Anon + Auth",
                         disabled="True", values=[], max=0)
-    hourNewAnonStats = dict(label=gettext("Anonymous"), values=[], max=0)
-    hourNewAuthStats = dict(label=gettext("Authenticated"),
+    hourNewAnonStats = dict(label="Anonymous", values=[], max=0)
+    hourNewAuthStats = dict(label="Authenticated",
                             values=[], max=0)
 
     hourNewStats['max'] = max_hours
@@ -457,15 +455,17 @@ def stats_format_hours(project_id, hours, hours_anon, hours_auth,
         if h in list(hours_anon.keys()):
             if (hours_anon[h] != 0):
                 tmph = (hours_anon[h] * 5) / max_hours
-                hourNewAnonStats['values'].append([int(h), hours_anon[h], tmph])
+                hourNewAnonStats['values'].append(
+                    [int(h), hours_anon[h], tmph])
             else:
                 hourNewAnonStats['values'].append([int(h), hours_anon[h], 0])
 
         # New Authenticated answers per hour
         if h in list(hours_auth.keys()):
             if (hours_auth[h] != 0):
                 tmph = (hours_auth[h] * 5) / max_hours
-                hourNewAuthStats['values'].append([int(h), hours_auth[h], tmph])
+                hourNewAuthStats['values'].append(
+                    [int(h), hours_auth[h], tmph])
             else:
                 hourNewAuthStats['values'].append([int(h), hours_auth[h], 0])
     return hourNewStats, hourNewAnonStats, hourNewAuthStats
@@ -496,7 +496,7 @@ def stats_format_users(project_id, users, anon_users, auth_users):
         top5_anon.append(dict(ip=u[0], tasks=u[1]))
 
     for u in auth_users:
-        sql = text('''SELECT name, fullname, restrict from "user" 
+        sql = text('''SELECT name, fullname, restrict from "user"
                    where id=:id and restrict=false;''')
         results = session.execute(sql, dict(id=u[0]))
         fullname = None
@@ -510,7 +510,7 @@ def stats_format_users(project_id, users, anon_users, auth_users):
             top5_auth.append(dict(name=name,
                                   fullname=fullname,
                                   tasks=u[1],
-                             restrict=restrict))
+                                  restrict=restrict))
 
     userAnonStats['top5'] = top5_anon[0:5]
     userAuthStats['top5'] = top5_auth
@@ -526,7 +526,6 @@ def update_stats(project_id, period='2 week'):
     users, anon_users, auth_users = stats_users(project_id, period)
     dates, dates_anon, dates_auth = stats_dates(project_id, period)
 
-
     sum(dates.values())
 
     sorted(iter(dates.items()), key=operator.itemgetter(0))

diff --git a/pybossa/core.py b/pybossa/core.py
@@ -20,7 +20,7 @@
 import logging
 import humanize
 from flask import Flask, url_for, request, render_template, \
-    flash, _app_ctx_stack, abort, redirect
+    flash, _app_ctx_stack, abort, redirect, request
 from flask_login import current_user
 from flask_babel import gettext
 from flask_assets import Bundle
@@ -276,9 +276,8 @@ def setup_babel(app):
 
     @babel.localeselector
     def _get_locale():
-        from flask import request
         locales = [l[0] for l in app.config.get('LOCALES')]
-        if current_user.is_authenticated:
+        if current_user and current_user.is_authenticated:
             lang = current_user.locale
         else:
             lang = request.cookies.get('language')

diff --git a/pybossa/signer/__init__.py b/pybossa/signer/__init__.py
@@ -17,7 +17,7 @@
 # along with PYBOSSA.  If not, see <http://www.gnu.org/licenses/>.
 
 from itsdangerous import URLSafeTimedSerializer
-from werkzeug import generate_password_hash, check_password_hash
+from werkzeug.security import generate_password_hash, check_password_hash
 
 
 class Signer(object):

diff --git a/pybossa/uploader/local.py b/pybossa/uploader/local.py
@@ -23,9 +23,9 @@
     * Local class: for uploading files to a local filesystem.
 
 """
-from pybossa.uploader import Uploader
 import os
-from werkzeug import secure_filename
+from pybossa.uploader import Uploader
+from werkzeug.utils import secure_filename
 
 
 class LocalUploader(Uploader):

diff --git a/setup.py b/setup.py
@@ -27,20 +27,20 @@
     "factory-boy==2.4.1",
     "Faker==1.0.1",
     "feedparser==5.2.1",
-    "Flask==1.0.2",
+    "Flask",
     "Flask-Assets==0.12",
-    "Flask-Babel==0.9",
-    "Flask-Cors==3.0.2",
+    "Flask-Babel>=0.9",
+    "Flask-Cors>=3.0.2",
     "Flask-DebugToolbar==0.10.1",
-    "Flask-HTTPAuth==3.2.4",
-    "flask-json-multidict==1.0.0",
-    "Flask-Login==0.4.1",
-    "Flask-Mail==0.9.1",
-    "Flask-Misaka==0.3.0",
-    "Flask-Plugins==1.6.1",
+    "Flask-HTTPAuth>=3.2.4",
+    "flask-json-multidict>=1.0.0",
+    "Flask-Login>=0.4.1",
+    "Flask-Mail>=0.9.1",
+    "Flask-Misaka>=1.0.0",
+    "Flask-Plugins>=1.6.1",
     "flask-profiler==1.6",
-    "Flask-SimpleLDAP==1.1.2",
-    "Flask-SQLAlchemy==2.3.2",
+    "Flask-SimpleLDAP>=1.1.2",
+    "Flask-SQLAlchemy>=2.3.2",
     "Flask-WTF>=0.9.5",
     "flatten-json==0.1.6",
     "google-api-python-client==1.5.5",
@@ -63,15 +63,15 @@
     "Mako==1.0.7",
     "Markdown==2.6.11",
     "MarkupSafe==1.1.0",
-    "misaka==1.0.2",
+    "misaka==2.1.1",
     "mock==2.0.0",
     "msgpack==0.6.0",
     "ndg-httpsclient==0.5.1",
     "netaddr==0.7.19",
     "netifaces==0.10.9",
     "nose==1.3.7",
     "nose-cov==1.6",
-    "numpy==1.15.4",
+    "numpy==1.20.1",
     "oauth2client==4.1.3",
     "oauthlib==2.1.0",
     "os-service-types==1.4.0",
@@ -80,7 +80,7 @@
     "oslo.serialization==2.28.1",
     "oslo.utils==3.39.0",
     "otpauth==1.0.1",
-    "pandas==0.23.4",
+    "pandas==1.2.3",
     "pbr>=1.10.0",
     "Pillow>=6.2.0",
     "prettytable==0.7.2",
@@ -132,7 +132,7 @@
     "urllib3==1.24.2",
     "validators==0.12.6",
     "webassets==0.12.1",
-    "Werkzeug==0.15.3",
+    "Werkzeug",
     "wrapt==1.10.11",
     "WTForms>=1.0.5",
     "WTForms-Components>=0.10.3",

diff --git a/test/default.py b/test/default.py
@@ -47,6 +47,13 @@ def decorated_function(*args, **kwargs):
             return f(*args, **kwargs)
     return decorated_function
 
+def with_request_context(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        with flask_app.test_request_context('/'):
+            return f(*args, **kwargs)
+    return decorated_function
+
 
 def with_context_settings(**kwargs):
     def decorator(f):

diff --git a/test/test_admin_export_users.py b/test/test_admin_export_users.py
@@ -53,7 +53,7 @@ def test_json_returns_all_users(self):
         self.signin()
 
         res = self.app.get('/admin/users/export?format=json',
-                            follow_redirects=True)
+                           follow_redirects=True)
         data = res.data
         json_data = json.loads(data)
 
@@ -70,7 +70,7 @@ def test_csv_contains_all_attributes(self):
         restricted = UserFactory.create(restrict=True, id=5000015)
 
         res = self.app.get('/admin/users/export?format=csv',
-                            follow_redirects=True)
+                           follow_redirects=True)
         data = res.data
 
         for attribute in self.exportable_attributes:
@@ -94,5 +94,5 @@ def test_csv_returns_all_users(self):
         data = res.data
         assert restricted.name not in str(data.decode('utf-8'))
         import pandas as pd
-        df = pd.DataFrame.from_csv(io.StringIO(data.decode('utf-8')))
+        df = pd.read_csv(io.StringIO(data.decode('utf-8')))
         assert df.shape[0] == 3, df.shape[0]
diff --git a/test/test_forms.py b/test/test_forms.py
@@ -20,7 +20,7 @@
 from nose.tools import raises, assert_raises
 from flask import current_app
 
-from default import Test, db, with_context
+from default import Test, db, with_context, with_request_context
 from pybossa.forms.forms import (RegisterForm, LoginForm, EMAIL_MAX_LENGTH,
     USER_NAME_MAX_LENGTH, USER_FULLNAME_MAX_LENGTH, BulkTaskLocalCSVImportForm,
     RegisterFormWithUserPrefMetadata, UserPrefMetadataForm)
@@ -134,15 +134,15 @@ def test_register_form_validates_with_valid_fields(self):
 
         assert form.validate()
 
-    @with_context
+    @with_request_context
     def test_register_form_unique_name(self):
         form = RegisterForm(**self.fill_in_data)
         user = UserFactory.create(name='mylion')
 
         assert not form.validate()
         assert "The user name is already taken" in form.errors['name'], form.errors
 
-    @with_context
+    @with_request_context
     def test_register_name_length(self):
         self.fill_in_data['name'] = 'a'
         form = RegisterForm(**self.fill_in_data)
@@ -151,15 +151,15 @@ def test_register_name_length(self):
         assert not form.validate()
         assert error_message in form.errors['name'], form.errors
 
-    @with_context
+    @with_request_context
     def test_register_name_allowed_chars(self):
         self.fill_in_data['name'] = '$#&amp;\/|'
         form = RegisterForm(**self.fill_in_data)
 
         assert not form.validate()
         assert "$#&\\/| and space symbols are forbidden" in form.errors['name'], form.errors
 
-    @with_context
+    @with_request_context
     def test_register_name_reserved_name(self):
         self.fill_in_data['name'] = 'signin'
 
@@ -168,15 +168,15 @@ def test_register_name_reserved_name(self):
         assert not form.validate()
         assert 'This name is used by the system.' in form.errors['name'], form.errors
 
-    @with_context
+    @with_request_context
     def test_register_form_unique_email(self):
         form = RegisterForm(**self.fill_in_data)
         user = UserFactory.create(email_addr='[email protected]')
 
         assert not form.validate()
         assert "Email is already taken" in form.errors['email_addr'], form.errors
 
-    @with_context
+    @with_request_context
     def test_register_email_length(self):
         self.fill_in_data['email_addr'] = ''
         form = RegisterForm(**self.fill_in_data)
@@ -193,7 +193,7 @@ def test_register_email_valid_format(self):
         assert not form.validate()
         assert "Invalid email address." in form.errors['email_addr'], form.errors
 
-    @with_context
+    @with_request_context
     def test_register_fullname_length(self):
         self.fill_in_data['fullname'] = 'a'
         form = RegisterForm(**self.fill_in_data)
@@ -202,15 +202,15 @@ def test_register_fullname_length(self):
         assert not form.validate()
         assert error_message in form.errors['fullname'], form.errors
 
-    @with_context
+    @with_request_context
     def test_register_password_required(self):
         self.fill_in_data['password'] = ''
         form = RegisterForm(**self.fill_in_data)
 
         assert not form.validate()
         assert "Password cannot be empty" in form.errors['password'], form.errors
 
-    @with_context
+    @with_request_context
     def test_register_password_missmatch(self):
         self.fill_in_data['confirm'] = 'badpasswd'
         form = RegisterForm(**self.fill_in_data)
@@ -309,14 +309,14 @@ def setUp(self):
                                     timezones=[("", ""), ("ACT", "Australia Central Time")],
                                     user_types=[("Researcher", "Researcher"), ("Analyst", "Analyst")])
 
-    @with_context
+    @with_request_context
     def test_register_form_with_upref_mdata_contains_fields(self):
         form = RegisterFormWithUserPrefMetadata()
 
         for field in self.fields:
             assert form.__contains__(field), 'Field %s is not in form' %field
 
-    @with_context
+    @with_request_context
     def test_register_form_with_upref_mdata_validates_with_valid_fields(self):
         import pybossa.core
         pybossa.core.upref_mdata_choices = self.upref_mdata_valid_choices