- added browser.py to extend webbot to use stealth

- fixed google login - now working as far as I can tell

Pipfile

@@ -18,6 +18,8 @@ requests = "*"
 webbot = "*"
 python-dateutil = "*"
 pathlib2 = "*"
+selenium-stealth= "*"
+selenium = "*"
 
 [requires]
 python_version = "3"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "9caab9704beb8ff3db73bfcde37140d67db8646ced74d49051ed489cc2448663"
+            "sha256": "fc0e2fd70a9660274318e874f607a1a93ed7e549023b60e1e9c08a48810b833e"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -53,7 +53,6 @@
                 "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
                 "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==4.0.0"
         },
         "idna": {
@@ -61,7 +60,6 @@
                 "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                 "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.10"
         },
         "pathlib2": {
@@ -93,14 +91,21 @@
                 "sha256:2d7131d7bc5a5b99a2d9b04aaf2612c411b03b8ca1b1ee8d3de5845a9be2cb3c",
                 "sha256:deaf32b60ad91a4611b98d8002757f29e6f2c2d5fcaf202e1c9ad06d6772300d"
             ],
+            "index": "pypi",
             "version": "==3.141.0"
         },
+        "selenium-stealth": {
+            "hashes": [
+                "sha256:b62da5452aa4a84f29a4dfb21a9696aff20788a7c570dd0b81bc04a940848b97"
+            ],
+            "index": "pypi",
+            "version": "==1.0.6"
+        },
         "six": {
             "hashes": [
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
                 "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.15.0"
         },
         "soupsieve": {
@@ -116,7 +121,6 @@
                 "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
                 "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
             "version": "==1.26.4"
         },
         "webbot": {
@@ -137,14 +141,29 @@
             "index": "pypi",
             "version": "==1.4.4"
         },
+        "atomicwrites": {
+            "hashes": [
+                "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197",
+                "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"
+            ],
+            "markers": "sys_platform == 'win32'",
+            "version": "==1.4.0"
+        },
         "attrs": {
             "hashes": [
                 "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
                 "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==20.3.0"
         },
+        "colorama": {
+            "hashes": [
+                "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b",
+                "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"
+            ],
+            "markers": "sys_platform == 'win32'",
+            "version": "==0.4.4"
+        },
         "distlib": {
             "hashes": [
                 "sha256:8c09de2c67b3e7deef7184574fc060ab8a793e7adbb183d942c389c8b13c52fb",
@@ -164,7 +183,6 @@
                 "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                 "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.10"
         },
         "iniconfig": {
@@ -214,7 +232,6 @@
                 "sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281",
                 "sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80"
             ],
-            "markers": "python_version >= '3.6'",
             "version": "==5.1.0"
         },
         "packaging": {
@@ -222,7 +239,6 @@
                 "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5",
                 "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==20.9"
         },
         "pluggy": {
@@ -230,7 +246,6 @@
                 "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
                 "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==0.13.1"
         },
         "py": {
@@ -238,7 +253,6 @@
                 "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3",
                 "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.10.0"
         },
         "pyparsing": {
@@ -246,7 +260,6 @@
                 "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
                 "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
             ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.4.7"
         },
         "pytest": {
@@ -267,11 +280,11 @@
         },
         "python-dotenv": {
             "hashes": [
-                "sha256:0c8d1b80d1a1e91717ea7d526178e3882732420b03f08afea0406db6402e220e",
-                "sha256:587825ed60b1711daea4832cf37524dfd404325b7db5e25ebe88c495c9f807a0"
+                "sha256:31d752f5b748f4e292448c9a0cac6a08ed5e6f4cefab85044462dcad56905cec",
+                "sha256:9fa413c37d4652d3fa02fea0ff465c384f5db75eab259c4fc5d0c5b8bf20edd4"
             ],
             "index": "pypi",
-            "version": "==0.15.0"
+            "version": "==0.16.0"
         },
         "pyyaml": {
             "hashes": [
@@ -305,7 +318,6 @@
                 "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6",
                 "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
             "version": "==5.4.1"
         },
         "six": {
@@ -313,7 +325,6 @@
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
                 "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.15.0"
         },
         "toml": {
@@ -321,7 +332,6 @@
                 "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
                 "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
             ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==0.10.2"
         },
         "tox": {
@@ -345,7 +355,6 @@
                 "sha256:12c3fcdae7b88ecf11fc0d3e6d77586549d4575a2ceee18e82eee75c1f626162",
                 "sha256:57095bf22fc0a2d99ee9674cdafebed0f3ba763018582450706f7d3a74fff599"
             ],
-            "markers": "python_version >= '3.5'",
             "version": "==4.1.1"
         },
         "virtualenv": {
@@ -353,7 +362,6 @@
                 "sha256:49ec4eb4c224c6f7dd81bb6d0a28a09ecae5894f4e593c89b0db0885f565a107",
                 "sha256:83f95875d382c7abafe06bd2a4cdd1b363e1bb77e02f155ebe8ac082a916b37c"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==20.4.3"
         },
         "wrapt": {

src/rival_regions_wrapper/authentication_handler.py

@@ -13,7 +13,7 @@ import pathlib2
 
 import requests
 import cfscrape
-from webbot.webbot import Browser
+from .browser import StealthBrowser as Browser
 from appdirs import user_data_dir
 
 
@@ -115,6 +115,7 @@ class AuthenticationHandler:
         self.login()
 
     def login(self):
+        self.remove_cookie(self.username)
         """Login user if needed"""
         LOGGER.info('"%s": start login, method: "%s"',
                     self.username, self.login_method)
@@ -141,7 +142,6 @@ class AuthenticationHandler:
 
             if self.login_method in method_dict:
                 browser = method_dict[self.login_method](browser, auth_text)
-                browser = method_dict[self.login_method](browser, auth_text)
             else:
                 LOGGER.info(
                         '"%s": Invalid login method "%s"',
@@ -209,45 +209,31 @@ class AuthenticationHandler:
         LOGGER.info('"%s": Login method Google', self.username)
         auth_text1 = auth_text.split('\t<a href="')
         auth_text2 = auth_text1[1].split('" class="sa')
+        time.sleep(1)
         browser.go_to(auth_text2[0])
 
-        number = 0
-
-        browser.get_screenshot_as_file("test_{}.png".format(number))
-        number += 1
-
         LOGGER.info('"%s": Typing in username', self.username)
         browser.type(self.username, into='Email')
 
-        browser.get_screenshot_as_file("test_{}.png".format(number))
-        number += 1
-
-        with open("test_1.html", 'w') as file_obj:
-            file_obj.write(browser.get_page_source())
-
         LOGGER.info('"%s": pressing next button', self.username)
         browser.click(css_selector="#next")
         time.sleep(2)
 
-        browser.get_screenshot_as_file("test_{}.png".format(number))
-        number += 1
-
         LOGGER.info('"%s": Typing in password', self.username)
         browser.type(self.password, css_selector="input")
 
-        with open("test_2.html", 'w') as file_obj:
-            file_obj.write(browser.get_page_source())
-
-        browser.get_screenshot_as_file("test_{}.png".format(number))
-        number += 1
-
         LOGGER.info('"%s": pressing sign in button', self.username)
         browser.click(css_selector="#submit")
-        time.sleep(2)
-
-        browser.get_screenshot_as_file("test_{}.png".format(number))
-        number += 1
+        time.sleep(3)
 
+        # Some why it wont click and login immediately. This seems to work
+        time.sleep(1)
+        browser.go_to(auth_text2[0])
+        time.sleep(1)
+        browser.go_to(auth_text2[0])
+        time.sleep(1)
+        browser.click(css_selector="#sa_add2 > div:nth-child(4) > a.sa_link.gogo > div")
+        time.sleep(3)
         return browser
 
     # IDK if this is working
@@ -295,6 +281,8 @@ class AuthenticationHandler:
                 raise FileNotFoundError # raise error as if file hadn't been found
         except FileNotFoundError:
             cookies = {username : {}}
+        if username not in cookies:
+            cookies[username] = {}
         LOGGER.info(cookies)
         for cookie in passed_cookies:
             cookies[username][cookie['name']] = {

src/rival_regions_wrapper/browser.py

@@ -0,0 +1,77 @@
+import os
+import errno
+from selenium_stealth import stealth
+from selenium import webdriver
+from selenium.webdriver.common.keys import Keys
+from webbot import Browser
+
+class StealthBrowser(Browser):
+    """
+
+    **Constructor**
+
+
+    :__init__(showWindow = True , proxy = None):
+        The constructor takes showWindow flag as argument which Defaults to False. If it is set to true , all browser happen without showing up any GUI window .
+
+        :Args:
+            - showWindow : If true , will run a headless browser without showing GUI window.
+            - proxy : Url of any optional proxy server.
+
+
+
+    Object attributes:  Key , errors
+
+    :Key:
+        - It contains the constants for all the special keys in the keyboard which can be used in the *press* method
+    errors:
+        - List containing all the errors which might have occurred during performing an action like click ,type etc.
+    """
+
+    def __init__(self, showWindow=True, proxy=None , downloadPath:str=None):
+        options = webdriver.ChromeOptions()
+        options.add_argument("--disable-dev-shm-usage")
+        options.add_argument("--no-sandbox")
+        options.add_argument('--disable-web-security')
+        options.add_argument('--allow-running-insecure-content')
+        options.add_argument("user-agent=DN")
+        options.add_experimental_option("excludeSwitches", ["enable-automation"])
+        options.add_experimental_option('useAutomationExtension', False)
+        if downloadPath is not None and isinstance(downloadPath,str):
+            absolutePath = os.path.abspath(downloadPath)
+            if(not os.path.isdir(absolutePath)):
+                raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), absolutePath)
+
+            options.add_experimental_option('prefs', {'download.default_directory' : absolutePath})
+
+        if proxy is not None and isinstance(proxy, str):
+            options.add_argument("--proxy-server={}".format(proxy))
+
+        if not showWindow:
+            options.headless = True
+
+        self.driver = webdriver.Chrome(options=options)
+        self.Key = Keys
+        self.errors = []
+        stealth(self.driver,
+                languages=["en-US", "en"],
+                vendor="Google Inc.",
+                platform="Win32",
+                webgl_vendor="Intel Inc.",
+                renderer="Intel Iris OpenGL Engine",
+                fix_hairline=True,
+                )
+
+        [setattr(self, function, getattr(self.driver, function)) for function in
+         ['add_cookie', 'delete_all_cookies', 'delete_cookie', 'execute_script', 'execute_async_script',
+          'fullscreen_window', 'get_cookie', 'get_cookies', 'get_log', 'get_network_conditions',
+          'get_screenshot_as_base64', 'get_screenshot_as_file', 'get_screenshot_as_png', 'get_window_position',
+          'get_window_rect', 'get_window_size', 'maximize_window', 'minimize_window', 'implicitly_wait', 'quit',
+          'refresh', 'save_screenshot', 'set_network_conditions', 'set_page_load_timeout', 'set_script_timeout',
+          'set_window_position', 'set_window_rect', 'start_client', 'start_session', 'stop_client', 'switch_to_alert']]
+
+
+
+
+
+