2026 update

Updates to readme documentation
Updates to application in crawl.py
Selenium module upgrade
Compatibility fix for running this Django 4.0 app with Python 13+ (created cgi.py)
Upgraded Web Crawling Pipeline (CI)

Author: kkamara

.env.example

@@ -5,6 +5,7 @@ CORS_ALLOWED_ORIGINS='http://localhost'
 TIME_ZONE='Europe/London'
 APP_ENV=development
 SIGNING_KEY=2de5c7b55db0e2bcbaac7a135e7b0cf340da8dba93f8c6d4f5ef0934887c834b
+CONNECTION=sqlite
 MARIADB_DATABASE=python_selenium
 MARIADB_USER=root
 MARIADB_PASSWORD=

.env.testing

@@ -5,6 +5,7 @@ CORS_ALLOWED_ORIGINS='http://localhost'
 TIME_ZONE='Europe/London'
 APP_ENV=testing
 SIGNING_KEY=2de5c7b55db0e2bcbaac7a135e7b0cf340da8dba93f8c6d4f5ef0934887c834b
+CONNECTION=mysql
 MARIADB_DATABASE=python_selenium
 MARIADB_USER=root
 MARIADB_PASSWORD=

.github/workflows/build.yml

@@ -14,11 +14,11 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.11]
+        python-version: [3.13]
 
     services:
       mysql:
-        image: mysql:5.7
+        image: mysql:8.0
         env:
           MYSQL_ROOT_PASSWORD: 'secret'
           MYSQL_DATABASE: 'python_selenium'
@@ -27,35 +27,23 @@ jobs:
         ports:
           - 3306/tcp
         options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
-      redis:
-        image: redis
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 6379:6379
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v7
       with:
         python-version: ${{ matrix.python-version }}
-    - uses: actions/checkout@v2
-    - uses: nanasess/setup-chromedriver@v2
+    - uses: nanasess/setup-chromedriver@v3
     - run: |
         export DISPLAY=:99
         chromedriver --url-base=/wd/hub &
+        # Xvfb is case-sensitive
         sudo Xvfb -ac :99 -screen 0 1280x1024x24 > /dev/null 2>&1 &
-    
     - name: Create environment file
       run: cp .env.testing .env
     - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
+      run: pip install -rvq --no-input requirements.txt
     - name: See your Python code do web browsing on your screen with GUI
       run: python manage.py crawl
     - name: Upload Screenshots

.gitignore

@@ -284,4 +284,5 @@ db.sqlite3
 migrations/
 nohup.out
 screenshots/
-tmp
+tmp
+/env

app/settings.py

@@ -15,7 +15,8 @@
 import os
 
 env = environ.Env(
-  DEBUG=(bool,False)
+  DEBUG=(bool,False),
+  CONNECTION=(str, None),
 )
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
@@ -38,11 +39,7 @@
 # SECURITY WARNING: keep the secret key used in production secret!
 SECRET_KEY = env('SECRET_KEY')
 
-SELENIUM_HEADLESS = env('SELENIUM_HEADLESS')
-if SELENIUM_HEADLESS == 'True':
-  SELENIUM_HEADLESS = True
-else:
-  SELENIUM_HEADLESS = False
+SELENIUM_HEADLESS = env.bool('SELENIUM_HEADLESS', default=False)
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = env('DEBUG')
@@ -227,17 +224,21 @@
 # Database
 # https://docs.djangoproject.com/en/4.0/ref/settings/#databases
 
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.mysql',
-        'NAME': env('MARIADB_DATABASE'),
-        'USER': env('MARIADB_USER'),
-        'PASSWORD': env('MARIADB_PASSWORD'),
-        'HOST': env('MARIADB_HOST'),
-        'PORT': env('MARIADB_PORT'),
+if env.str("CONNECTION") is None or env.str("CONNECTION") == "sqlite":
+    DATABASES = {
+        "default": {
+            "ENGINE": "django.db.backends.sqlite3",
+            "NAME": BASE_DIR / "db.sqlite3",
+        }
+    }
+else:
+    DATABASES = {
+        "default": {
+            "ENGINE": "django.db.backends.mysql",
+            "NAME": env("MARIADB_DATABASE"),
+            "USER": env("MARIADB_USER"),
+            "PASSWORD": env("MARIADB_PASSWORD"),
+            "HOST": env("MARIADB_HOST"),
+            "PORT": env("MARIADB_PORT"),
+        }
     }
-}
-
-if 'production' == APP_ENV:
-  import django_on_heroku
-  django_on_heroku.settings(locals())

cgi.py

@@ -0,0 +1,24 @@
+"""Compatibility shim for Python 3.13+ where stdlib cgi was removed.
+
+Only the API used by this project stack (Django 4.0.x) is implemented.
+"""
+
+from email.message import Message
+
+
+def parse_header(line):
+    """Parse a Content-type like header.
+
+    Returns (main_value, params_dict), matching the old cgi.parse_header shape.
+    """
+    message = Message()
+    message["content-type"] = line
+
+    main_value = message.get_content_type()
+    params = {}
+
+    # Skip the first item, which repeats the content type itself.
+    for key, value in message.get_params()[1:]:
+        params[key] = value
+
+    return main_value, params

readme.md

@@ -6,8 +6,14 @@
 
 * [Important note:](#note)
 
+* [Requirements](#requirements)
+
 * [Installation](#installation)
 
+    * [Configure This Project](#configure-this-project)
+
+    * [Run Your Selenium Server Jar File](#run-your-selenium-server-jar-file)
+
 * [Usage](#usage)
 
 * [Using Docker](#using-docker)
@@ -30,54 +36,55 @@
 
 ## Important note: <a name="note"></a>
 
-Before you try to scrape any website, go through its robots.txt file. You can access it via `domainname/robots.txt`. There, you will see a list of pages allowed and disallowed for scraping. You should not violate any terms of service of any website you scrape. 
+Before you try to scrape any website, go through its robots.txt file. You can access it via `domainname/robots.txt`. There, you will see a list of pages allowed and disallowed for scraping. You should not violate any terms of service of any website you scrape.
+
+## Requirements
+
+* [Tested using Python 3.13](https://www.python.org)
+* [Chromedriver](https://developer.chrome.com/docs/chromedriver)
+* [Selenium Server JAR file](https://www.selenium.dev/documentation/grid/getting_started)
 
 ## Installation
 
+#### Configure This Project
+
 ```bash
 cp .env.example .env
-python3 -m venv env && \
+python -m venv env && \
   source env/bin/activate
 pip install -r requirements.txt
-python3 manage.py migrate
+python manage.py makemigrations 
+python manage.py migrate
 ```
 
-#### Add chromedriver to Path
+#### Run Your Selenium Server JAR File
 
-Make sure Chromedriver is installed and added to your environment Path.
+Locate where you downloaded your Selenium Server JAR file in the [requirements](#requirements) step and run the following.
 
 ```bash
-# chromedriver_mac64
-# chromedriver_win32
-# See https://chromedriver.storage.googleapis.com
-# for drivers list.
-wget https://chromedriver.storage.googleapis.com/2.37/chromedriver_linux64.zip
-unzip chromedriver_linux64.zip
-sudo mv chromedriver /usr/local/bin/chromedriver
-chromedriver --version
+java -jar selenium-server-[version].jar standalone --override-max-sessions true --max-sessions 10
 ```
 
+[CLI options in the Selenium Grid](https://www.selenium.dev/documentation/grid/configuration/cli_options/).
+
 ## Usage
 
 [XPath cheat sheet](https://devhints.io/xpath).
 
-Update the command at [crawl.py](https://github.com/kkamara/python-selenium/blob/main/seleniumpy/management/commands/crawl.py)
+Update the command at [crawl.py](https://github.com/kkamara/python-selenium/blob/main/seleniumpy/management/commands/crawl.py) to perform your instructions in web scraping.
 
 ```bash
-alias py="python3"
-py manage.py crawl
+python manage.py crawl
 ```
 
-If you still need help installing and running the app check out the readme at https://github.com/kkamara/python-react-boilerplate which is the base system for this python-selenium app.
-
 ## Using Docker?
 
 ```bash
 alias compose='docker-compose -f local.yml'
 compose build
 compose up
 # Automated runs with Docker:
-# compose up --build -d && python3 manage.py crawl
+# compose up --build -d && python manage.py crawl
 ```
 
 ## iPython Django Shell

requirements.txt

@@ -1,6 +1,5 @@
-selenium==3.141.0
+selenium==4.45.0
 asgiref==3.5.0
-certifi==2021.10.8
 chardet==4.0.0
 charset-normalizer==2.0.12
 coreapi==2.3.3
@@ -31,13 +30,13 @@ PyJWT==1.7.1
 PyMySQL==1.0.2
 python3-openid==3.2.0
 pytz==2022.1
-requests==2.27.1
+requests==2.34.2
 requests-oauthlib==1.3.1
 six==1.16.0
 social-auth-app-django==4.0.0
 social-auth-core==4.0.2
 sqlparse==0.4.2
 uritemplate==4.1.1
-urllib3==1.26.9
 whitenoise==6.0.0
 zipp==3.8.0
+ipython~=9.14.1

seleniumpy/management/commands/crawl.py

@@ -1,70 +1,72 @@
 from django.core.management.base import BaseCommand, CommandError
 from django.conf import settings
 
-import os
-import json
 import time
 import logging
 import random
 
 from selenium import webdriver
 from selenium.webdriver.common.by import By
-from selenium.webdriver.common.keys import Keys
+
 
 class Command(BaseCommand):
-    help = 'help text'
+    help = "help text"
 
     def handle(self, *args, **options):
-        '''
+        """
         The Selenium Python Documentation:
             https://www.selenium.dev/documentation/
-        '''
+        """
         try:
             chrome_options = webdriver.ChromeOptions()
-            chrome_options.add_argument('--disable-extensions')
-            chrome_options.add_argument('--disable-gpu')
-            chrome_options.add_argument('--no-sandbox')
-            chrome_options.add_argument('--disable-blink-features=AutomationControlled')
+            chrome_options.add_argument("--disable-extensions")
+            chrome_options.add_argument("--disable-gpu")
+            chrome_options.add_argument("--no-sandbox")
+            chrome_options.add_argument("--disable-blink-features=AutomationControlled")
             user_agent = self.get_random_user_agent()
-            logging.info(f'Using User Agent : '+user_agent)
-            chrome_options.add_argument('--user-agent='+user_agent)
+            logging.info(f"Using User Agent : {user_agent}")
+            chrome_options.add_argument(f"--user-agent={user_agent}")
             # Use SELENIUM_HEADLESS in .env to remove GUI.
-            if settings.SELENIUM_HEADLESS == True or settings.APP_ENV == 'testing':
-                chrome_options.add_argument('--headless')
-            if settings.APP_ENV == 'testing':
-                chrome_options.add_argument('--disable-dev-shm-usage')
-            browser = webdriver.Chrome(options=chrome_options)
-            
-            browser.get('https://www.kelvinkamara.com')
+            if settings.SELENIUM_HEADLESS or settings.APP_ENV == "testing":
+                chrome_options.add_argument("--headless")
+            if settings.APP_ENV == "testing":
+                chrome_options.add_argument("--disable-dev-shm-usage")
+            browser = webdriver.Remote(
+                command_executor="http://127.0.0.1:4444/wd/hub", options=chrome_options
+            )
 
-            self.screenshot(browser, name='debug')
+            browser.get("https://www.kelvinkamara.com")
 
-            if 'Thisisnotinpagesource.' in browser.page_source:
-                raise RuntimeError('We were detected.')            
-            time.sleep(1)
+            self.screenshot(browser, name="debug")
 
-            browser.find_element(
-                By.XPATH, "//a[@id='contact-me-link']").click()
-            
+            if "Thisisnotinpagesource." in browser.page_source:
+                raise RuntimeError("We were detected.")
             time.sleep(1)
 
+            browser.find_element(By.XPATH, "//a[@id='contact-me-link']").click()
+
+            time.sleep(2)
+
             browser.quit()
-            self.stdout.write(self.style.SUCCESS('Success'))
+            self.stdout.write(self.style.SUCCESS("Success"))
         except Exception as e:
             try:
                 browser.quit()
-            except:
+            except Exception:
                 pass
             raise CommandError(str(e))
 
-    def screenshot(self, browser, el=None, name='example'):
-        browser.save_screenshot(f'./screenshots/{name}.png')
+    def screenshot(self, browser, el=None, name="example"):
+        browser.save_screenshot(f"./screenshots/{name}.png")
         if el:
-            el_text = browser.execute_script('return arguments[0].innerText', el)
+            el_text = browser.execute_script("return arguments[0].innerText", el)
             logging.info(el_text)
 
     def get_random_user_agent(self):
         user_agents = [
-            'Mozilla/5.0 (BB10; Touch) AppleWebKit/537.1+ (KHTML, like Gecko) Version/10.0.0.1337 Mobile Safari/537.1+', 'Mozilla/5.0 (PlayBook; U; RIM Tablet OS 2.1.0; en-US) AppleWebKit/536.2+ (KHTML, like Gecko) Version/7.2.1.0 Safari/536.2+', 'Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.0.0.187 Mobile Safari/534.11+']
+            "Mozilla/5.0 (BB10; Touch) AppleWebKit/537.1+ (KHTML, like Gecko) Version/10.0.0.1337 Mobile Safari/537.1+",
+            "Mozilla/5.0 (PlayBook; U; RIM Tablet OS 2.1.0; en-US) AppleWebKit/536.2+ (KHTML, like Gecko) Version/7.2.1.0 Safari/536.2+",
+            "Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.0.0.187 Mobile Safari/534.11+",
+        ]
         key = random.randint(0, 2)
-        return user_agents[key]
+        return user_agents[key]