Extract URLs with Protocol, Domain & Path Parsing (Python)

💻 Code Snippet — July 21, 2026

Extract URLs with Protocol, Domain & Path Parsing (Python)

The usual re.findall(r'http\S+', text) is broken for production — it misses bare domains, captures trailing punctuation, and can't handle www.-prefixed URLs.

import re
from collections import namedtuple

URLMatch = namedtuple('URLMatch', ['full', 'scheme', 'host', 'path'])

def extract_urls(text):
    pattern = r'(?:https?://|www\.)\S+'
    raw = re.findall(pattern, text, re.IGNORECASE)
    # Strip trailing punctuation
    cleaned = [re.sub(r'[.,;:!?)]+$', '', u) for u in raw]
    # Deduplicate preserving order
    seen, result = set(), []
    for url in cleaned:
        if url not in seen:
            seen.add(url)
            scheme = re.match(r'(https?)://', url, re.I)
            host = re.search(r'://([^/?#]+)', url)
            path = re.search(r'(?:://[^/?#]+)?(/[^?#]*)', url)
            result.append(URLMatch(
                full=url,
                scheme=scheme.group(1) if scheme else 'http',
                host=host.group(1) if host else url.replace('www.', '').split('/')[0],
                path=path.group(1) if path else '/'
            ))
    return result

# Usage
urls = extract_urls("Visit https://example.com/path?q=1 and www.test.org/page")
for u in urls:
    print(f"{u.scheme}://{u.host}{u.path}")

Why bookmark this: Handles all 4 URL forms, strips trailing punctuation, deduplicates, returns structured results.

💡 Pro Tip: Add urllib.parse.urlparse() for full query/fragment parsing in production.