Upgrades and fixes before releasing 4.0 (#418)

Co-authored-by: alainrouillon <31690441+alainrouillon@users.noreply.github.com>
Co-authored-by: Suhun Han <ssut@ssut.me>
Co-authored-by: fcoagz <grihardware@gmail.com>
Co-authored-by: Suhun Han <ssut@users.noreply.github.com>

.github/workflows/ci.yml

@@ -0,0 +1,27 @@
+name: Python tests
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install tox tox-gh-actions
+    - name: Test with tox
+      run: tox
+
+

Pipfile

@@ -4,7 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-httpx = "==0.14.1"
+httpx = {extras = ["http2"], version = "~=0.23"}
 
 [dev-packages]
 coveralls = "*"
@@ -13,4 +13,4 @@ coveralls = "*"
 sphinx = "*"
 
 [requires]
-python_version = ">=3.6"
+python_version = "^3.6"

README.rst

@@ -106,6 +106,21 @@ URLs are provided, it then randomly chooses a domain.
           'translate.google.co.kr',
         ])
 
+Customize service URL to point to standard api
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Considering translate.google.<domain> url services use the webapp requiring a token, 
+you can prefer to use the direct api than does not need any token to process.
+It can solve your problems of unstable token providing processes (refer to issue #234)
+
+.. code:: python
+
+    >>> from googletrans import Translator
+    >>> translator = Translator(service_urls=[
+          'translate.googleapis.com'
+        ])
+
+
 Advanced Usage (Bulk)
 ~~~~~~~~~~~~~~~~~~~~~
 

googletrans/__init__.py

@@ -1,6 +1,6 @@
 """Free Google Translate API for Python. Translates totally free of charge."""
 __all__ = 'Translator',
-__version__ = '3.0.0'
+__version__ = '3.4.0'
 
 
 from googletrans.client import Translator

googletrans/client.py

@@ -6,6 +6,7 @@
 """
 import random
 import typing
+import re
 
 import httpcore
 import httpx
@@ -14,6 +15,7 @@
 from googletrans import urls, utils
 from googletrans.gtoken import TokenAcquirer
 from googletrans.constants import (
+    DEFAULT_CLIENT_SERVICE_URLS,
     DEFAULT_USER_AGENT, LANGCODES, LANGUAGES, SPECIAL_CASES,
     DEFAULT_RAISE_EXCEPTION, DUMMY_DATA
 )
@@ -29,6 +31,7 @@ class Translator:
 
     :param service_urls: google translate url list. URLs will be used randomly.
                          For example ``['translate.google.com', 'translate.google.co.kr']``
+                         To preferably use the non webapp api, service url should be translate.googleapis.com
     :type service_urls: a sequence of strings
 
     :param user_agent: the User-Agent header to send when making requests.
@@ -49,9 +52,9 @@ class Translator:
     :type raise_exception: boolean
     """
 
-    def __init__(self, service_urls=None, user_agent=DEFAULT_USER_AGENT,
+    def __init__(self, service_urls=DEFAULT_CLIENT_SERVICE_URLS, user_agent=DEFAULT_USER_AGENT,
                  raise_exception=DEFAULT_RAISE_EXCEPTION,
-                 proxies: typing.Dict[str, httpcore.SyncHTTPTransport] = None,
+                 proxies: typing.Dict[str, httpcore.AsyncHTTPProxy] = None,
                  timeout: Timeout = None,
                  http2=True):
 
@@ -63,12 +66,29 @@ def __init__(self, service_urls=None, user_agent=DEFAULT_USER_AGENT,
             'User-Agent': user_agent,
         })
 
+        self.service_urls = ['translate.google.com']
+        self.client_type = 'webapp'
+        self.token_acquirer = TokenAcquirer(
+            client=self.client, host=self.service_urls[0])
+
         if timeout is not None:
             self.client.timeout = timeout
 
-        self.service_urls = service_urls or ['translate.google.com']
-        self.token_acquirer = TokenAcquirer(
-            client=self.client, host=self.service_urls[0])
+        if service_urls:
+            #default way of working: use the defined values from user app
+            self.service_urls = service_urls
+            self.client_type = 'webapp'
+            self.tok1en_acquirer = TokenAcquirer(
+                client=self.client, host=self.service_urls[0])
+
+            #if we have a service url pointing to client api we force the use of it as defaut client
+            for t in enumerate(service_urls):
+                api_type = re.search('googleapis',service_urls[0])
+                if (api_type):
+                    self.service_urls = ['translate.googleapis.com']
+                    self.client_type = 'gtx'
+                    break
+
         self.raise_exception = raise_exception
 
     def _pick_service_url(self):
@@ -77,8 +97,11 @@ def _pick_service_url(self):
         return random.choice(self.service_urls)
 
     def _translate(self, text, dest, src, override):
-        token = self.token_acquirer.do(text)
-        params = utils.build_params(query=text, src=src, dest=dest,
+        token = 'xxxx' #dummy default value here as it is not used by api client
+        if self.client_type == 'webapp':
+            token = self.token_acquirer.do(text)
+
+        params = utils.build_params(client=self.client_type, query=text, src=src, dest=dest,
                                     token=token, override=override)
 
         url = urls.TRANSLATE.format(host=self._pick_service_url())
@@ -276,4 +299,4 @@ def detect(self, text, **kwargs):
             pass
         result = Detected(lang=src, confidence=confidence, response=response)
 
-        return result
+        return result

googletrans/constants.py

@@ -1,5 +1,9 @@
 DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
 
+DEFAULT_CLIENT_SERVICE_URLS = (
+    'translate.googleapis.com',
+)
+
 DEFAULT_SERVICE_URLS = ('translate.google.ac', 'translate.google.ad', 'translate.google.ae',
                         'translate.google.al', 'translate.google.am', 'translate.google.as',
                         'translate.google.at', 'translate.google.az', 'translate.google.ba',
@@ -21,21 +25,21 @@
                         'translate.google.com.ai', 'translate.google.com.ar', 'translate.google.com.au',
                         'translate.google.com.bd', 'translate.google.com.bh', 'translate.google.com.bn',
                         'translate.google.com.bo', 'translate.google.com.br', 'translate.google.com.bz',
-                        'translate.google.com.co', 'translate.google.com.cu', 'translate.google.com.cy', 
-                        'translate.google.com.do', 'translate.google.com.ec', 'translate.google.com.eg', 
-                        'translate.google.com.et', 'translate.google.com.fj', 'translate.google.com.gh', 
-                        'translate.google.com.gi', 'translate.google.com.gt', 'translate.google.com.hk', 
-                        'translate.google.com.jm', 'translate.google.com.kh', 'translate.google.com.kw', 
-                        'translate.google.com.lb', 'translate.google.com.ly', 'translate.google.com.mm', 
-                        'translate.google.com.mt', 'translate.google.com.mx', 'translate.google.com.my', 
-                        'translate.google.com.na', 'translate.google.com.ng', 'translate.google.com.ni', 
-                        'translate.google.com.np', 'translate.google.com.om', 'translate.google.com.pa', 
-                        'translate.google.com.pe', 'translate.google.com.pg', 'translate.google.com.ph', 
-                        'translate.google.com.pk', 'translate.google.com.pr', 'translate.google.com.py', 
-                        'translate.google.com.qa', 'translate.google.com.sa', 'translate.google.com.sb', 
-                        'translate.google.com.sg', 'translate.google.com.sl', 'translate.google.com.sv', 
-                        'translate.google.com.tj', 'translate.google.com.tr', 'translate.google.com.tw', 
-                        'translate.google.com.ua', 'translate.google.com.uy', 'translate.google.com.vc', 
+                        'translate.google.com.co', 'translate.google.com.cu', 'translate.google.com.cy',
+                        'translate.google.com.do', 'translate.google.com.ec', 'translate.google.com.eg',
+                        'translate.google.com.et', 'translate.google.com.fj', 'translate.google.com.gh',
+                        'translate.google.com.gi', 'translate.google.com.gt', 'translate.google.com.hk',
+                        'translate.google.com.jm', 'translate.google.com.kh', 'translate.google.com.kw',
+                        'translate.google.com.lb', 'translate.google.com.ly', 'translate.google.com.mm',
+                        'translate.google.com.mt', 'translate.google.com.mx', 'translate.google.com.my',
+                        'translate.google.com.na', 'translate.google.com.ng', 'translate.google.com.ni',
+                        'translate.google.com.np', 'translate.google.com.om', 'translate.google.com.pa',
+                        'translate.google.com.pe', 'translate.google.com.pg', 'translate.google.com.ph',
+                        'translate.google.com.pk', 'translate.google.com.pr', 'translate.google.com.py',
+                        'translate.google.com.qa', 'translate.google.com.sa', 'translate.google.com.sb',
+                        'translate.google.com.sg', 'translate.google.com.sl', 'translate.google.com.sv',
+                        'translate.google.com.tj', 'translate.google.com.tr', 'translate.google.com.tw',
+                        'translate.google.com.ua', 'translate.google.com.uy', 'translate.google.com.vc',
                         'translate.google.com.vn', 'translate.google.com', 'translate.google.cv',
                         'translate.google.cz', 'translate.google.de', 'translate.google.dj',
                         'translate.google.dk', 'translate.google.dm', 'translate.google.dz',
@@ -65,9 +69,9 @@
                         'translate.google.st', 'translate.google.td', 'translate.google.tg',
                         'translate.google.tk', 'translate.google.tl', 'translate.google.tm',
                         'translate.google.tn', 'translate.google.to', 'translate.google.tt',
-                        'translate.google.us', 'translate.google.vg', 'translate.google.vu', 
+                        'translate.google.us', 'translate.google.vg', 'translate.google.vu',
                         'translate.google.ws')
-                        
+
 SPECIAL_CASES = {
     'ee': 'et',
 }

googletrans/gtoken.py

@@ -58,11 +58,14 @@ def _update(self):
             self.tkk = raw_tkk.group(1)
             return
 
-        # this will be the same as python code after stripping out a reserved word 'var'
-        code = self.RE_TKK.search(r.text).group(1).replace('var ', '')
-        # unescape special ascii characters such like a \x3d(=)
-        code = code.encode().decode('unicode-escape')
-
+        code = self.RE_TKK.search(r.text)
+
+        if code is not None:
+            # this will be the same as python code after stripping out a reserved word 'var'
+            code = code.group(1).replace('var ', '')
+            # unescape special ascii characters such like a \x3d(=)
+            code = code.encode().decode('unicode-escape')
+
         if code:
             tree = ast.parse(code)
             visit_return = False

googletrans/utils.py

@@ -3,9 +3,9 @@
 import re
 
 
-def build_params(query, src, dest, token, override):
+def build_params(client,query, src, dest, token, override):
     params = {
-        'client': 'webapp',
+        'client': client,
         'sl': src,
         'tl': dest,
         'hl': dest,

setup.py

@@ -58,7 +58,7 @@ def install():
         packages=find_packages(exclude=['docs', 'tests']),
         keywords='google translate translator',
         install_requires=[
-            'httpx==0.13.3',
+            'httpx[http2]>=0.23,<0.27.3',
         ],
         python_requires= '>=3.6',
         tests_require=[

tests/test_client.py

@@ -19,6 +19,15 @@ def test_bind_multiple_service_urls():
     assert translator.translate('test', dest='ko')
     assert translator.detect('Hello')
 
+def test_api_service_urls():
+    service_urls = ['translate.googleapis.com']
+
+    translator = Translator(service_urls=service_urls)
+    assert translator.service_urls == service_urls
+
+    assert translator.translate('test', dest='ko')
+    assert translator.detect('Hello')
+
 
 def test_source_language(translator):
     result = translator.translate('안녕하세요.')
@@ -38,7 +47,7 @@ def test_pronunciation_issue_175(translator):
 
 def test_latin_to_english(translator):
     result = translator.translate('veritas lux mea', src='la', dest='en')
-    assert result.text == 'The truth is my light'
+    assert result.text == 'truth is my light'
 
 
 def test_unicode(translator):
@@ -53,7 +62,7 @@ def test_emoji(translator):
 
 def test_language_name(translator):
     result = translator.translate(u'Hello', src='ENGLISH', dest='iRiSh')
-    assert result.text == u'Dia dhuit'
+    assert result.text == u'Dia duit'
 
 
 def test_language_name_with_space(translator):
@@ -75,39 +84,44 @@ def test_special_chars(translator):
 
 
 def test_translate_list(translator):
-    args = (['test', 'exam'], 'ko', 'en')
+    args = (['test', 'exam', 'exam paper'], 'ko', 'en')
     translations = translator.translate(*args)
 
-    assert translations[0].text == u'테스트'
+    assert translations[0].text == u'시험'
     assert translations[1].text == u'시험'
+    assert translations[2].text == u'시험지'
 
 
 def test_detect_language(translator):
     ko = translator.detect(u'한국어')
     en = translator.detect('English')
     rubg = translator.detect('тест')
+    russ = translator.detect('привет')
 
     assert ko.lang == 'ko'
     assert en.lang == 'en'
-    assert rubg.lang == ['ru', 'bg']
+    assert rubg.lang == 'mk'
+    assert russ.lang == 'ru'
+    #'bg']
 
 
 def test_detect_list(translator):
-    items = [u'한국어', ' English', 'тест']
+    items = [u'한국어', ' English', 'тест', 'привет']
 
     result = translator.detect(items)
 
     assert result[0].lang == 'ko'
     assert result[1].lang == 'en'
-    assert result[2].lang == ['ru', 'bg']
+    assert result[2].lang == 'mk'
+    assert result[3].lang == 'ru'
 
 
 def test_src_in_special_cases(translator):
-    args = ('Tere', 'en', 'ee')
+    args = ('tere', 'en', 'ee')
 
     result = translator.translate(*args)
 
-    assert result.text in ('Hello', 'Hi,')
+    assert result.text in ('hello', 'hi,')
 
 
 def test_src_not_in_supported_languages(translator):
@@ -122,7 +136,7 @@ def test_dest_in_special_cases(translator):
 
     result = translator.translate(*args)
 
-    assert result.text == 'Tere'
+    assert result.text == 'tere'
 
 
 def test_dest_not_in_supported_languages(translator):

tests/test_utils.py

@@ -28,6 +28,7 @@ def test_rshift():
 
 def test_build_params_with_override():
     params = utils.build_params(
+        client='',
         query='',
         src='',
         dest='',

tox.ini

@@ -1,5 +1,14 @@
 [tox]
-envlist = py36,py37,py38,pypy3
+envlist = py36,py37,py38,pypy3,py39,py311,py312
+
+[gh-actions]
+python =
+    3.7: py37
+    3.8: py38
+    3.9: py39
+    3.10: py310
+    3.11: py311
+    3.12: py312
 
 [testenv]
 deps=