Upgrade dependencies to support up to python 3.13

Author: gsaint

code-env/python/desc.json

@@ -3,7 +3,11 @@
         "PYTHON36",
         "PYTHON37",
         "PYTHON38",
-        "PYTHON39"
+        "PYTHON39",
+        "PYTHON310",
+        "PYTHON311",
+        "PYTHON312",
+        "PYTHON313"
     ],
     "forceConda": false,
     "installCorePackages": true,

code-env/python/spec/requirements.txt

@@ -1,5 +1,6 @@
 emoji==0.6.0
-fastcore==1.3.1
+fastcore==1.3.1; python_version < '3.10'
+fastcore==1.12.6; python_version >= '3.10'
 jieba==0.42.1
 langid==1.1.6
 pycld3==0.22; python_version < '3.10'
@@ -9,7 +10,8 @@ pymorphy2-dicts-uk==2.4.1.1.1460299261
 pymorphy2==0.9.1
 pyvi==0.1
 regex==2021.4.4
-spacy[lookups,ja,th]==2.3.5
+spacy[lookups,ja,th]==2.3.5; python_version < '3.10'
+spacy[lookups,ja,th]==3.8.11; python_version >= '3.10'
 symspellpy==6.7.0
 tqdm==4.60.0
 sudachipy==0.6.0; python_version == '3.6'

python-lib/language_detector.py

@@ -145,7 +145,7 @@ def detect_languages_df(self, df: pd.DataFrame, text_column: AnyStr) -> pd.DataF
         self.column_descriptions = {}
         for k, v in self.COLUMN_DESCRIPTIONS.items():
             self.column_descriptions[generate_unique(k, df.keys(), text_column)] = v
-        doc_iterator = (doc for _, doc in df[text_column].astype(str).iteritems())
+        doc_iterator = (doc for _, doc in df[text_column].astype(str).items())
         output_df = df.copy()
         with ThreadPoolExecutor(max_workers=self.NUM_THREADS) as executor:
             lang_output_tuple_list = list(executor.map(self.detect_language_doc, doc_iterator))

tests/python/unit/requirements.txt

@@ -1,4 +1,6 @@
-pandas~=1.0
-numpy==1.26.4
+pandas~=1.0; python_version < '3.13'
+pandas~=2.0; python_version >= '3.13'
+numpy==1.26.4; python_version < '3.13'
+numpy==2.4.1; python_version >= '3.13'
 pytest~=6.2
 allure-pytest~=2.8

tests/python/unit/test_language_detector.py

@@ -25,7 +25,7 @@
 OUTPUT_DF["input_text"] = INPUT_DF["input_text"]
 OUTPUT_DF["input_text_language_code"] = ["", "es", "fr", "en", "ja"]
 OUTPUT_DF["input_text_language_name"] = ["", "Spanish", "French", "English", "Japanese"]
-OUTPUT_DF["input_text_language_score"] = [np.NaN, np.NaN, 1.0, 1.0, 1.0]
+OUTPUT_DF["input_text_language_score"] = [np.nan, np.nan, 1.0, 1.0, 1.0]
 
 def test_language_detector():
     detector = LanguageDetector(minimum_score=0.2, fallback_language="es")