<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="https://media.rss.com/style.xsl"?>
<rss xmlns:podcast="https://podcastindex.org/namespace/1.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:psc="http://podlove.org/simple-chapters" xmlns:atom="http://www.w3.org/2005/Atom" xml:lang="en" version="2.0">
  <channel>
    <title><![CDATA[DEWA Podcasts]]></title>
    <link>https://rss.com/podcasts/dewa-news</link>
    <atom:link href="https://media.rss.com/dewa-news/feed.xml" rel="self" type="application/rss+xml"/>
    <atom:link rel="hub" href="https://pubsubhubbub.appspot.com/"/>
    <description><![CDATA[
Since its establishment, Dubai Electricity and Water Authority (DEWA) has achieved major milestones, ranking among the world’s top utilities. It serves over 1.2 million customers in Dubai, maintaining high standards of efficiency, reliability and availability. In April 2022, DEWA was listed on the Dubai Financial Market as its largest company, with a market value of AED 124 billion (USD 33.8 billion). DEWA outperforms major European and American utilities. In 2024, electricity transmission and distribution losses fell to 2%, compared to 6–7% in Europe and the USA, while water network losses dropped to 4.5% versus 15% in North America. DEWA also achieved the world’s lowest electricity Customer Minutes Lost (CML), at just 0.94 minutes per customer, far below the 15-minute average of leading European utilities.
]]></description>
    <generator>RSS.com 2026.320.183334</generator>
    <lastBuildDate>Wed, 01 Apr 2026 08:58:34 GMT</lastBuildDate>
    <language>en</language>
    <copyright><![CDATA[Dubai Electricity & Water Authority]]></copyright>
    <itunes:image href="https://media.rss.com/dewa-news/20250523_040515_a31ff5ab754aa1346615c1a67cc806b6.jpg"/>
    <podcast:guid>5a0dd3bd-33fb-5a3d-9bf5-39af9fc875a6</podcast:guid>
    <image>
      <url>https://media.rss.com/dewa-news/20250523_040515_a31ff5ab754aa1346615c1a67cc806b6.jpg</url>
      <title>DEWA Podcasts</title>
      <link>https://rss.com/podcasts/dewa-news</link>
    </image>
    <podcast:locked>yes</podcast:locked>
    <podcast:license>Dubai Electricity & Water Authority</podcast:license>
    <itunes:author>Dubai Electricity & Water Authority</itunes:author>
    <itunes:owner>
      <itunes:name>Dubai Electricity & Water Authority</itunes:name>
    </itunes:owner>
    <itunes:explicit>false</itunes:explicit>
    <itunes:type>episodic</itunes:type>
    <itunes:category text="Government"/>
    <itunes:category text="News"/>
    <podcast:medium>podcast</podcast:medium>
    <item>
      <title><![CDATA[Campaign held under the patronage of HH Sheikh Hamdan bin Mohammed bin Rashid Al Maktoum DEWA supports Dubai Autism Center’s 20th Autism Awareness Campaign as Sustainability Partner]]></title>
      <itunes:title><![CDATA[Campaign held under the patronage of HH Sheikh Hamdan bin Mohammed bin Rashid Al Maktoum DEWA supports Dubai Autism Center’s 20th Autism Awareness Campaign as Sustainability Partner]]></itunes:title>
      <description><![CDATA[
Campaign held under the patronage of HH Sheikh Hamdan bin Mohammed bin Rashid Al Maktoum DEWA supports Dubai Autism Center’s 20th Autism Awareness Campaign as Sustainability Partner
]]></description>
      <link>https://rss.com/podcasts/dewa-news/2679905</link>
      <enclosure url="https://content.rss.com/episodes/331008/2679905/dewa-news/2026_04_01_08_57_46_9e4716e2-4221-4269-9b02-a9f6cb3705f8.mp3" length="2550848" type="audio/mpeg"/>
      <guid isPermaLink="false">824f15ed-0470-4ed0-be40-057753ec87ac</guid>
      <itunes:duration>159</itunes:duration>
      <itunes:episodeType>full</itunes:episodeType>
      <itunes:explicit>false</itunes:explicit>
      <pubDate>Wed, 01 Apr 2026 08:58:31 GMT</pubDate>
      <itunes:image href="https://media.rss.com/dewa-news/ep_cover_20260401_080443_9c81bdfae2419bd2f8db44aea2e759e6.png"/>
      <podcast:transcript url="https://transcripts.rss.com/331008/2679905/transcript" type="text/vtt"/>
    </item>
    <item>
      <title><![CDATA[تحت رعاية سمو الشيخ حمدان بن محمد بن راشد آل مكتوم هيئة كهرباء ومياه دبي شريك الاستدامة للحملة السنوية العشرين للتوعية بالتوحد]]></title>
      <itunes:title><![CDATA[تحت رعاية سمو الشيخ حمدان بن محمد بن راشد آل مكتوم هيئة كهرباء ومياه دبي شريك الاستدامة للحملة السنوية العشرين للتوعية بالتوحد]]></itunes:title>
      <description><![CDATA[
تحت رعاية سمو الشيخ حمدان بن محمد بن راشد آل مكتوم

هيئة كهرباء ومياه دبي شريك الاستدامة للحملة السنوية العشرين للتوعية بالتوحد
]]></description>
      <link>https://rss.com/podcasts/dewa-news/2679700</link>
      <enclosure url="https://content.rss.com/episodes/331008/2679700/dewa-news/2026_04_01_06_52_58_b2751f61-91c6-44ff-941d-0bfa9ab0928a.mp3" length="3245915" type="audio/mpeg"/>
      <guid isPermaLink="false">637d2c1e-557f-4038-8e89-145c19595d8c</guid>
      <itunes:duration>202</itunes:duration>
      <itunes:episodeType>full</itunes:episodeType>
      <itunes:explicit>false</itunes:explicit>
      <pubDate>Wed, 01 Apr 2026 06:57:35 GMT</pubDate>
      <itunes:image href="https://media.rss.com/dewa-news/ep_cover_20260401_060440_90635ec1f011f77c4366610380cc2169.png"/>
      <podcast:transcript url="https://transcripts.rss.com/331008/2679700/transcript" type="text/vtt"/>
    </item>
    <item>
      <title><![CDATA[عمومية "إمباور" توافق على توزيع 437.5 مليون درهم أرباحاً مدعومة بأداء قوي للمؤسسة ]]></title>
      <itunes:title><![CDATA[عمومية "إمباور" توافق على توزيع 437.5 مليون درهم أرباحاً مدعومة بأداء قوي للمؤسسة ]]></itunes:title>
      <description><![CDATA[
عمومية "إمباور" توافق على توزيع 437.5 مليون درهم أرباحاً مدعومة بأداء قوي للمؤسسة
]]></description>
      <link>https://rss.com/podcasts/dewa-news/2663613</link>
      <enclosure url="https://content.rss.com/episodes/331008/2663613/dewa-news/2026_03_27_05_37_58_0c54a26d-bbf0-46f1-90ba-47b7a6bf9569.mp3" length="2415557" type="audio/mpeg"/>
      <guid isPermaLink="false">bda596fe-e69f-4e6b-8297-2c031cbbb59d</guid>
      <itunes:duration>150</itunes:duration>
      <itunes:episodeType>full</itunes:episodeType>
      <itunes:explicit>false</itunes:explicit>
      <pubDate>Fri, 27 Mar 2026 05:39:22 GMT</pubDate>
      <itunes:image href="https://media.rss.com/dewa-news/ep_cover_20260326_010340_3e4018dcdff6f2e0980fad21686e40b3.jpg"/>
      <podcast:transcript url="https://transcripts.rss.com/331008/2663613/transcript" type="text/vtt"/>
    </item>
    <item>
      <title><![CDATA[Empower Annual General Meeting approves cash dividends of AED 437.5 million supported by the company's strong performance]]></title>
      <itunes:title><![CDATA[Empower Annual General Meeting approves cash dividends of AED 437.5 million supported by the company's strong performance]]></itunes:title>
      <description><![CDATA[
Empower Annual General Meeting approves cash dividends of AED 437.5 million supported by the company's strong performance
]]></description>
      <link>https://rss.com/podcasts/dewa-news/2663615</link>
      <enclosure url="https://content.rss.com/episodes/331008/2663615/dewa-news/2026_03_27_05_38_21_1d5ef8f9-f4ea-4ceb-a94b-e8fe17a41e7c.mp3" length="2969771" type="audio/mpeg"/>
      <guid isPermaLink="false">02ec3905-dc66-4838-a67c-4185a500b525</guid>
      <itunes:duration>185</itunes:duration>
      <itunes:episodeType>full</itunes:episodeType>
      <itunes:explicit>false</itunes:explicit>
      <pubDate>Fri, 27 Mar 2026 05:39:17 GMT</pubDate>
      <itunes:image href="https://media.rss.com/dewa-news/ep_cover_20260326_010343_1b58506dd014e48564fe51f107e263a5.jpg"/>
      <podcast:transcript url="https://transcripts.rss.com/331008/2663615/transcript" type="text/vtt"/>
    </item>
    <item>
      <title><![CDATA[DEWA marks Reading Month 2026 with series of events promoting intellectual and cultural growth]]></title>
      <itunes:title><![CDATA[DEWA marks Reading Month 2026 with series of events promoting intellectual and cultural growth]]></itunes:title>
      <description><![CDATA[
DEWA marks Reading Month 2026 with series of events promoting intellectual and cultural growth
]]></description>
      <link>https://rss.com/podcasts/dewa-news/2662730</link>
      <enclosure url="https://content.rss.com/episodes/331008/2662730/dewa-news/2026_03_26_04_23_39_d03e428e-a7ac-4f80-86e5-142627467146.mp3" length="2812073" type="audio/mpeg"/>
      <guid isPermaLink="false">7b111d73-b24f-4dbf-ab5d-684c32474286</guid>
      <itunes:duration>175</itunes:duration>
      <itunes:episodeType>full</itunes:episodeType>
      <itunes:explicit>false</itunes:explicit>
      <pubDate>Thu, 26 Mar 2026 04:24:20 GMT</pubDate>
      <itunes:image href="https://media.rss.com/dewa-news/ep_cover_20260326_040337_da324f019a1c635f3f0833e6a0c2a62c.jpg"/>
      <podcast:transcript url="https://transcripts.rss.com/331008/2662730/transcript" type="text/vtt"/>


Now I have everything. Let me completely rewrite the scraper with the exact selectors and RSS feed parsing:


Bash

python3 -c "
from pathlib import Path
import re
html = Path('debug_news.html').read_text()

# Check the description element - data-vl vs innerText
idx = html.find('teaser__copy')
print(html[idx:idx+400])
"

Check teaser description element structure


teaser__copy" data-vl="As a part of its social responsibility, Dubai Electricity and Water Authority (DEWA) is supporting the Dubai Autism Centre’s 20th Autism Awareness Campaign as Sustainability Partner. The annual campaign is held under the patronage of HH Sheikh Hamdan bin Mohammed bin Rashid Al Maktoum, Crown Prince of Dubai, Deputy Prime Minister and Minister of Defence of the UAE, and Chair


Write

"""
DEWA PR Tracker — Scraper
  - Press Releases: Playwright (Chromium) — iterates all years via year filter
  - Podcasts:       Direct RSS feed parse from media.rss.com (no browser needed)

Usage:
  python scraper.py              # normal run
  python scraper.py --debug      # visible browser + saves debug HTML/screenshots
  python scraper.py --visible    # visible browser only
"""
import argparse
import re
import xml.etree.ElementTree as ET
from datetime import datetime
from email.utils import parsedate_to_datetime
from pathlib import Path
from urllib.request import urlopen, Request

from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout

import database as db
from config import (
    DEWA_NEWS_URL, DEWA_BASE_URL,
    SCRAPER_HEADLESS, SCRAPER_TIMEOUT_MS, SCRAPER_WAIT_AFTER_LOAD_MS,
)

RSS_FEED_URL = "https://media.rss.com/dewa-news/feed.xml"

# Years available on the DEWA news page
SCRAPE_YEARS = ["2019", "2020", "2021", "2022", "2023", "2024", "2025", "2026"]

# ── Utilities ─────────────────────────────────────────────────────────────────

def normalise_url(href: str) -> str:
    if not href:
        return ""
    href = href.strip()
    if href.startswith("http"):
        return href
    if href.startswith("/"):
        return DEWA_BASE_URL + href
    return href

def parse_dewa_date(raw: str) -> str:
    """Parse DEWA date format '30 March 2026' → '2026-03-30'."""
    if not raw:
        return None
    raw = raw.strip()
    try:
        return datetime.strptime(raw, "%d %B %Y").strftime("%Y-%m-%d")
    except ValueError:
        pass
    try:
        return datetime.strptime(raw, "%d %b %Y").strftime("%Y-%m-%d")
    except ValueError:
        pass
    m = re.search(r"(\d{4})", raw)
    return m.group(1) + "-01-01" if m else None

def seconds_to_mmss(secs) -> str:
    try:
        s = int(secs)
        return f"{s // 60}:{s % 60:02d}"
    except (TypeError, ValueError):
        return str(secs) if secs else None

# ── Press Release scraper (Playwright) ───────────────────────────────────────

def extract_items_from_page(page) -> list:
    """Extract all .m9-teaser--news_all items currently visible on the page."""
    items = page.query_selector_all(".m9-teaser--news_all")
    results = []
    for item in items:
        try:
            # Link — the whole card is an
            link_el = item.query_selector("a.teaser__link")
            if not link_el:
                link_el = item.query_selector("a")
            href  = link_el.get_attribute("href") if link_el else ""
            url   = normalise_url(href)

            # Title
            title_el = item.query_selector("h4.teaser__title")
            if not title_el:
                title_el = item.query_selector("h2, h3, h4, h5, .title")
            title = title_el.inner_text().strip() if title_el else ""

            if not title or not url:
                continue

            # Date — "30 March 2026"
            date_el = item.query_selector("h6.teaser__date")
            raw_date = date_el.inner_text().strip() if date_el else ""
            date = parse_dewa_date(raw_date)

            # Description — stored in data-vl attribute of the

            desc_el = item.query_selector("p.teaser__copy")
            description = None
            if desc_el:
                description = (
                    desc_el.get_attribute("data-vl")
                    or desc_el.inner_text().strip()
                    or None
                )

            results.append({
                "title": title,
                "url": url,
                "date": date,
                "description": description,
                "category": None,
            })
        except Exception as e:
            print(f"  [warn] Skipping item: {e}")
    return results

def scrape_press_releases(page, debug: bool = False) -> list:
    print(f"[scraper] → News page: {DEWA_NEWS_URL}")
    page.goto(DEWA_NEWS_URL, timeout=SCRAPER_TIMEOUT_MS, wait_until="networkidle")
    page.wait_for_timeout(SCRAPER_WAIT_AFTER_LOAD_MS)

    if debug:
        page.screenshot(path="debug_news.png", full_page=True)
        Path("debug_news.html").write_text(page.content(), encoding="utf-8")
        print("[debug] Saved debug_news.png + debug_news.html")

    all_results = []
    seen_urls   = set()

    for year in SCRAPE_YEARS:
        print(f"  [scraper] Selecting year {year}…", end=" ", flush=True)
        try:
            # Select the year in the dropdown
            year_select = page.query_selector("select#form-field-years, select[name='year']")
            if not year_select:
                print("year selector not found, skipping")
                continue
            year_select.select_option(value=year)
            page.wait_for_timeout(2500)   # wait for results to reload

            # Also reset month to "All Months" to get all items in the year
            month_select = page.query_selector("select#form-field-months, select[name='month']")
            if month_select:
                month_select.select_option(value="")
                page.wait_for_timeout(2000)

            items = extract_items_from_page(page)
            new_items = [i for i in items if i["url"] not in seen_urls]
            for i in new_items:
                seen_urls.add(i["url"])
            all_results.extend(new_items)
            print(f"{len(new_items)} new items ({len(items)} total on page)")

        except Exception as e:
            print(f"error: {e}")
            continue

    print(f"[scraper] Total press releases extracted: {len(all_results)}")
    return all_results

# ── Podcast scraper (RSS feed) ────────────────────────────────────────────────

def scrape_podcasts_from_rss() -> list:
    print(f"[scraper] → Fetching RSS feed: {RSS_FEED_URL}")
    try:
        req = Request(
            RSS_FEED_URL,
            headers={"User-Agent": "Mozilla/5.0 (compatible; DEWATracker/1.0)"}
        )
        with urlopen(req, timeout=30) as resp:
            raw_xml = resp.read()
    except Exception as e:
        print(f"[scraper] RSS fetch error: {e}")
        return []

    # Parse XML
    ns = {
        "itunes":  "http://www.itunes.com/dtds/podcast-1.0.dtd",
        "podcast": "https://podcastindex.org/namespace/1.0",
        "content": "http://purl.org/rss/1.0/modules/content/",
    }

    try:
        root = ET.fromstring(raw_xml)
    except ET.ParseError as e:
        print(f"[scraper] XML parse error: {e}")
        return []

    channel = root.find("channel")
    if not channel:
        return []

    results = []
    for item in channel.findall("item"):
        try:
            title = (item.findtext("title") or "").strip()
            link  = (item.findtext("link")  or "").strip()

            if not title or not link:
                continue

            # Date — RFC 822 format e.g. "Wed, 01 Apr 2026 08:58:31 GMT"
            pub_date_raw = item.findtext("pubDate") or ""
            date = None
            if pub_date_raw:
                try:
                    date = parsedate_to_datetime(pub_date_raw).strftime("%Y-%m-%d")
                except Exception:
                    pass

            # MP3 URL from <enclosure>
            enclosure = item.find("enclosure")
            audio_url = enclosure.get("url") if enclosure is not None else None

            # Duration (seconds → mm:ss)
            dur_raw  = item.findtext("itunes:duration", namespaces=ns) or ""
            duration = seconds_to_mmss(dur_raw) if dur_raw else None

            # Description
            desc_raw = item.findtext("description") or ""
            desc_raw = re.sub(r"<[^>]+>", "", desc_raw).strip()
            description = desc_raw or None

            # Language detection (Arabic title?)
            is_arabic = bool(re.search(r'[\u0600-\u06FF]', title))
            language  = "ar" if is_arabic else "en"

            results.append({
                "title":       title,
                "url":         link,
                "audio_url":   audio_url,
                "date":        date,
                "duration":    duration,
                "description": description,
                "language":    language,
            })
        except Exception as e:
            print(f"  [warn] Skipping podcast item: {e}")
            continue

    print(f"[scraper] Total podcast episodes from RSS: {len(results)}")
    return results

# ── Match podcasts → press releases by title ──────────────────────────────────

def normalise_title(t: str) -> str:
    """Lowercase, strip punctuation for fuzzy matching."""
    t = t.lower()
    t = re.sub(r"[^\w\s]", " ", t)
    t = re.sub(r"\s+", " ", t).strip()
    return t

def match_podcasts_to_prs(podcasts: list, pr_id_map: dict) -> list:
    """
    pr_id_map: { url: (id, normalised_title) }
    Match by:
    1. URL slug — rss.com episode URL may contain matching text
    2. Title substring overlap (>60% word overlap)
    """
    pr_titles = [(pid, ntitle) for _, (pid, ntitle) in pr_id_map.items()]

    matched = []
    for pod in podcasts:
        pr_id = None
        pod_norm = normalise_title(pod["title"])
        pod_words = set(pod_norm.split())

        best_score = 0
        best_pid   = None
        for pid, pr_norm in pr_titles:
            pr_words = set(pr_norm.split())
            if not pr_words:
                continue
            overlap = len(pod_words & pr_words) / len(pr_words)
            if overlap > best_score:
                best_score = overlap
                best_pid   = pid

        if best_score >= 0.5:
            pr_id = best_pid

        matched.append({**pod, "pr_id": pr_id})

    linked = sum(1 for p in matched if p["pr_id"])
    print(f"[scraper] Podcasts linked to PRs: {linked}/{len(matched)}")
    return matched

# ── Main ──────────────────────────────────────────────────────────────────────

def run_scrape(debug: bool = False, headless: bool = None) -> tuple[int, int]:
    if headless is None:
        headless = SCRAPER_HEADLESS
    if debug:
        headless = False

    db.init_db()

    prs_saved  = 0
    pods_saved = 0
    status     = "success"
    notes      = ""

    try:
        # ── 1. Press Releases ─────────────────────────────────────────────────
        with sync_playwright() as pw:
            browser = pw.chromium.launch(
                headless=headless,
                args=["--disable-blink-features=AutomationControlled"],
            )
            ctx = browser.new_context(
                user_agent=(
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
                    "AppleWebKit/537.36 (KHTML, like Gecko) "
                    "Chrome/122.0.0.0 Safari/537.36"
                ),
                viewport={"width": 1440, "height": 900},
            )
            page = ctx.new_page()
            pr_data = scrape_press_releases(page, debug=debug)
            browser.close()

        pr_id_map = {}
        for pr in pr_data:
            pid = db.upsert_press_release(
                title=pr["title"],
                url=pr["url"],
                date=pr["date"],
                category=pr.get("category"),
                description=pr.get("description"),
            )
            pr_id_map[pr["url"]] = (pid, normalise_title(pr["title"]))
            prs_saved += 1

        # ── 2. Podcasts from RSS feed ─────────────────────────────────────────
        pod_data = scrape_podcasts_from_rss()
        pod_data = match_podcasts_to_prs(pod_data, pr_id_map)
        for pod in pod_data:
            db.upsert_podcast(
                title=pod["title"],
                url=pod["url"],
                pr_id=pod.get("pr_id"),
                audio_url=pod.get("audio_url"),
                date=pod.get("date"),
                duration=pod.get("duration"),
            )
            pods_saved += 1

    except Exception as e:
        status = "error"
        notes  = str(e)
        print(f"[scraper] ERROR: {e}")

    db.log_scrape(prs_saved, pods_saved, status, notes)
    print(f"\n✅ Done — {prs_saved} PRs, {pods_saved} podcasts saved. Status: {status}")
    if notes:
        print(f"   Notes: {notes}")

    return prs_saved, pods_saved

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="DEWA PR & Podcast Scraper")
    parser.add_argument("--debug",   action="store_true", help="Visible browser + save debug files")
    parser.add_argument("--visible", action="store_true", help="Visible browser only")
    args = parser.parse_args()

    run_scrape(debug=args.debug, headless=not args.visible and SCRAPER_HEADLESS)


The file /Users/khalidbinkalban/Projects/dewa-pr-tracker/scraper.py has been updated successfully.

"""
DEWA PR Tracker — Scraper
  - Press Releases: Playwright (Chromium) — iterates all years via year filter
  - Podcasts:       Direct RSS feed parse from media.rss.com (no browser needed)

Usage:
  python scraper.py              # normal run
  python scraper.py --debug      # visible browser + saves debug HTML/screenshots
  python scraper.py --visible    # visible browser only
"""
import argparse
import re
import xml.etree.ElementTree as ET
from datetime import datetime
from email.utils import parsedate_to_datetime
from pathlib import Path
from urllib.request import urlopen, Request

from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout

import database as db
from config import (
    DEWA_NEWS_URL, DEWA_BASE_URL,
    SCRAPER_HEADLESS, SCRAPER_TIMEOUT_MS, SCRAPER_WAIT_AFTER_LOAD_MS,
)

RSS_FEED_URL = "https://media.rss.com/dewa-news/feed.xml"

# Years available on the DEWA news page
SCRAPE_YEARS = ["2019", "2020", "2021", "2022", "2023", "2024", "2025", "2026"]

# ── Utilities ─────────────────────────────────────────────────────────────────

def normalise_url(href: str) -> str:
    if not href:
        return ""
    href = href.strip()
    if href.startswith("http"):
        return href
    if href.startswith("/"):
        return DEWA_BASE_URL + href
    return href

def parse_dewa_date(raw: str) -> str:
    """Parse DEWA date format '30 March 2026' → '2026-03-30'."""
    if not raw:
        return None
    raw = raw.strip()
    try:
        return datetime.strptime(raw, "%d %B %Y").strftime("%Y-%m-%d")
    except ValueError:
        pass
    try:
        return datetime.strptime(raw, "%d %b %Y").strftime("%Y-%m-%d")
    except ValueError:
        pass
    m = re.search(r"(\d{4})", raw)
    return m.group(1) + "-01-01" if m else None

def seconds_to_mmss(secs) -> str:
    try:
        s = int(secs)
        return f"{s // 60}:{s % 60:02d}"
    except (TypeError, ValueError):
        return str(secs) if secs else None

# ── Press Release scraper (Playwright) ───────────────────────────────────────

def extract_items_from_page(page) -> list:
    """Extract all .m9-teaser--news_all items currently visible on the page."""
    items = page.query_selector_all(".m9-teaser--news_all")
    results = []
    for item in items:
        try:
            # Link — the whole card is an
            link_el = item.query_selector("a.teaser__link")
            if not link_el:
                link_el = item.query_selector("a")
            href  = link_el.get_attribute("href") if link_el else ""
            url   = normalise_url(href)

            # Title
            title_el = item.query_selector("h4.teaser__title")
            if not title_el:
                title_el = item.query_selector("h2, h3, h4, h5, .title")
            title = title_el.inner_text().strip() if title_el else ""

            if not title or not url:
                continue

            # Date — "30 March 2026"
            date_el = item.query_selector("h6.teaser__date")
            raw_date = date_el.inner_text().strip() if date_el else ""
            date = parse_dewa_date(raw_date)

            # Description — stored in data-vl attribute of the

            desc_el = item.query_selector("p.teaser__copy")
            description = None
            if desc_el:
                description = (
                    desc_el.get_attribute("data-vl")
                    or desc_el.inner_text().strip()
                    or None
                )

            results.append({
                "title": title,
                "url": url,
                "date": date,
                "description": description,
                "category": None,
            })
        except Exception as e:
            print(f"  [warn] Skipping item: {e}")
    return results

def scrape_press_releases(page, debug: bool = False) -> list:
    print(f"[scraper] → News page: {DEWA_NEWS_URL}")
    page.goto(DEWA_NEWS_URL, timeout=SCRAPER_TIMEOUT_MS, wait_until="networkidle")
    page.wait_for_timeout(SCRAPER_WAIT_AFTER_LOAD_MS)

    if debug:
        page.screenshot(path="debug_news.png", full_page=True)
        Path("debug_news.html").write_text(page.content(), encoding="utf-8")
        print("[debug] Saved debug_news.png + debug_news.html")

    all_results = []
    seen_urls   = set()

    for year in SCRAPE_YEARS:
        print(f"  [scraper] Selecting year {year}…", end=" ", flush=True)
        try:
            # Select the year in the dropdown
            year_select = page.query_selector("select#form-field-years, select[name='year']")
            if not year_select:
                print("year selector not found, skipping")
                continue
            year_select.select_option(value=year)
            page.wait_for_timeout(2500)   # wait for results to reload

            # Also reset month to "All Months" to get all items in the year
            month_select = page.query_selector("select#form-field-months, select[name='month']")
            if month_select:
                month_select.select_option(value="")
                page.wait_for_timeout(2000)

            items = extract_items_from_page(page)
            new_items = [i for i in items if i["url"] not in seen_urls]
            for i in new_items:
                seen_urls.add(i["url"])
            all_results.extend(new_items)
            print(f"{len(new_items)} new items ({len(items)} total on page)")

        except Exception as e:
            print(f"error: {e}")
            continue

    print(f"[scraper] Total press releases extracted: {len(all_results)}")
    return all_results

# ── Podcast scraper (RSS feed) ────────────────────────────────────────────────

def scrape_podcasts_from_rss() -> list:
    print(f"[scraper] → Fetching RSS feed: {RSS_FEED_URL}")
    try:
        req = Request(
            RSS_FEED_URL,
            headers={"User-Agent": "Mozilla/5.0 (compatible; DEWATracker/1.0)"}
        )
        with urlopen(req, timeout=30) as resp:
            raw_xml = resp.read()
    except Exception as e:
        print(f"[scraper] RSS fetch error: {e}")
        return []

    # Parse XML
    ns = {
        "itunes":  "http://www.itunes.com/dtds/podcast-1.0.dtd",
        "podcast": "https://podcastindex.org/namespace/1.0",
        "content": "http://purl.org/rss/1.0/modules/content/",
    }

    try:
        root = ET.fromstring(raw_xml)
    except ET.ParseError as e:
        print(f"[scraper] XML parse error: {e}")
        return []

    channel = root.find("channel")
    if not channel:
        return []

    results = []
    for item in channel.findall("item"):
        try:
            title = (item.findtext("title") or "").strip()
            link  = (item.findtext("link")  or "").strip()

            if not title or not link:
                continue

            # Date — RFC 822 format e.g. "Wed, 01 Apr 2026 08:58:31 GMT"
            pub_date_raw = item.findtext("pubDate") or ""
            date = None
            if pub_date_raw:
                try:
                    date = parsedate_to_datetime(pub_date_raw).strftime("%Y-%m-%d")
                except Exception:
                    pass

            # MP3 URL from <enclosure>
            enclosure = item.find("enclosure")
            audio_url = enclosure.get("url") if enclosure is not None else None

            # Duration (seconds → mm:ss)
            dur_raw  = item.findtext("itunes:duration", namespaces=ns) or ""
            duration = seconds_to_mmss(dur_raw) if dur_raw else None

            # Description
            desc_raw = item.findtext("description") or ""
            desc_raw = re.sub(r"<[^>]+>", "", desc_raw).strip()
            description = desc_raw or None

            # Language detection (Arabic title?)
            is_arabic = bool(re.search(r'[\u0600-\u06FF]', title))
            language  = "ar" if is_arabic else "en"

            results.append({
                "title":       title,
                "url":         link,
                "audio_url":   audio_url,
                "date":        date,
                "duration":    duration,
                "description": description,
                "language":    language,
            })
        except Exception as e:
            print(f"  [warn] Skipping podcast item: {e}")
            continue

    print(f"[scraper] Total podcast episodes from RSS: {len(results)}")
    return results

# ── Match podcasts → press releases by title ──────────────────────────────────

def normalise_title(t: str) -> str:
    """Lowercase, strip punctuation for fuzzy matching."""
    t = t.lower()
    t = re.sub(r"[^\w\s]", " ", t)
    t = re.sub(r"\s+", " ", t).strip()
    return t

def match_podcasts_to_prs(podcasts: list, pr_id_map: dict) -> list:
    """
    pr_id_map: { url: (id, normalised_title) }
    Match by:
    1. URL slug — rss.com episode URL may contain matching text
    2. Title substring overlap (>60% word overlap)
    """
    pr_titles = [(pid, ntitle) for _, (pid, ntitle) in pr_id_map.items()]

    matched = []
    for pod in podcasts:
        pr_id = None
        pod_norm = normalise_title(pod["title"])
        pod_words = set(pod_norm.split())

        best_score = 0
        best_pid   = None
        for pid, pr_norm in pr_titles:
            pr_words = set(pr_norm.split())
            if not pr_words:
                continue
            overlap = len(pod_words & pr_words) / len(pr_words)
            if overlap > best_score:
                best_score = overlap
                best_pid   = pid

        if best_score >= 0.5:
            pr_id = best_pid

        matched.append({**pod, "pr_id": pr_id})

    linked = sum(1 for p in matched if p["pr_id"])
    print(f"[scraper] Podcasts linked to PRs: {linked}/{len(matched)}")
    return matched

# ── Main ──────────────────────────────────────────────────────────────────────

def run_scrape(debug: bool = False, headless: bool = None) -> tuple[int, int]:
    if headless is None:
        headless = SCRAPER_HEADLESS
    if debug:
        headless = False

    db.init_db()

    prs_saved  = 0
    pods_saved = 0
    status     = "success"
    notes      = ""

    try:
        # ── 1. Press Releases ─────────────────────────────────────────────────
        with sync_playwright() as pw:
            browser = pw.chromium.launch(
                headless=headless,
                args=["--disable-blink-features=AutomationControlled"],
            )
            ctx = browser.new_context(
                user_agent=(
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
                    "AppleWebKit/537.36 (KHTML, like Gecko) "
                    "Chrome/122.0.0.0 Safari/537.36"
                ),
                viewport={"width": 1440, "height": 900},
            )
            page = ctx.new_page()
            pr_data = scrape_press_releases(page, debug=debug)
            browser.close()

        pr_id_map = {}
        for pr in pr_data:
            pid = db.upsert_press_release(
                title=pr["title"],
                url=pr["url"],
                date=pr["date"],
                category=pr.get("category"),
                description=pr.get("description"),
            )
            pr_id_map[pr["url"]] = (pid, normalise_title(pr["title"]))
            prs_saved += 1

        # ── 2. Podcasts from RSS feed ─────────────────────────────────────────
        pod_data = scrape_podcasts_from_rss()
        pod_data = match_podcasts_to_prs(pod_data, pr_id_map)
        for pod in pod_data:
            db.upsert_podcast(
                title=pod["title"],
                url=pod["url"],
                pr_id=pod.get("pr_id"),
                audio_url=pod.get("audio_url"),
                date=pod.get("date"),
                duration=pod.get("duration"),
            )
            pods_saved += 1

    except Exception as e:
        status = "error"
        notes  = str(e)
        print(f"[scraper] ERROR: {e}")

    db.log_scrape(prs_saved, pods_saved, status, notes)
    print(f"\n✅ Done — {prs_saved} PRs, {pods_saved} podcasts saved. Status: {status}")
    if notes:
        print(f"   Notes: {notes}")

    return prs_saved, pods_saved

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="DEWA PR & Podcast Scraper")
    parser.add_argument("--debug",   action="store_true", help="Visible browser + save debug files")
    parser.add_argument("--visible", action="store_true", help="Visible browser only")
    args = parser.parse_args()

    run_scrape(debug=args.debug, headless=not args.visible and SCRAPER_HEADLESS)

-DEWA PR Tracker — Playwright-based scraper

-Debug mode:    python scraper.py --debug   (saves HTML + screenshot, opens browser)

+  - Press Releases: Playwright (Chromium) — iterates all years via year filter

+  - Podcasts:       Direct RSS feed parse from media.rss.com (no browser needed)

+  python scraper.py              # normal run

+  python scraper.py --debug      # visible browser + saves debug HTML/screenshots

+  python scraper.py --visible    # visible browser only

+from email.utils import parsedate_to_datetime

+from urllib.request import urlopen, Request

-    DEWA_NEWS_URL, DEWA_PODCAST_URL, DEWA_BASE_URL,

-# ── Selector candidates (tried in order until one matches) ───────────────────

+# Years available on the DEWA news page

-DATE_SELECTORS   = ["time", ".date", "[class*='date']", "[datetime]", "span.label"]

-    """Try to parse a messy date string into YYYY-MM-DD."""

-            return datetime.strptime(raw, fmt).strftime("%Y-%m-%d")

+        return datetime.strptime(raw, "%d %B %Y").strftime("%Y-%m-%d")

+        return datetime.strptime(raw, "%d %b %Y").strftime("%Y-%m-%d")

return m.group(1) + "-01-01" if m else None

-            el = element.query_selector(sel)

+        return str(secs) if secs else None

-# ── Core scraping logic ───────────────────────────────────────────────────────

+# ── Press Release scraper (Playwright) ───────────────────────────────────────

-def scrape_press_releases(page, debug: bool = False):

-    print(f"[scraper] Loading news page → {DEWA_NEWS_URL}")

-    page.goto(DEWA_NEWS_URL, timeout=SCRAPER_TIMEOUT_MS, wait_until="networkidle")

-    page.wait_for_timeout(SCRAPER_WAIT_AFTER_LOAD_MS)

-        page.screenshot(path="debug_news.png", full_page=True)

-        Path("debug_news.html").write_text(page.content(), encoding="utf-8")

-        print("[debug] Saved debug_news.png + debug_news.html")

-    # Auto-scroll to trigger lazy-loading

-        page.evaluate("window.scrollBy(0, window.innerHeight)")

-    # Try "Load more" / pagination buttons

-                "button:has-text('Load more'), button:has-text('Show more'), "

-                "a:has-text('Load more'), a:has-text('Next'), .pagination .next"

-            if btn and btn.is_visible():

-                page.wait_for_timeout(2000)

-    # Find the container with news items

-        found = page.query_selector_all(sel)

-            print(f"[scraper] Using selector '{sel}' → {len(found)} items found")

-        print("[scraper] WARNING: No items found with known selectors. Run with --debug to inspect HTML.")

+def extract_items_from_page(page) -> list:

+    """Extract all .m9-teaser--news_all items currently visible on the page."""

+    items = page.query_selector_all(".m9-teaser--news_all")

-            title_el = first_match(item, TITLE_SELECTORS)

-            title = title_el.inner_text().strip()

-            href  = title_el.get_attribute("href") or ""

+            # Link — the whole card is an <a class="teaser__link">

+            link_el = item.query_selector("a.teaser__link")

-                href = link_el.get_attribute("href") if link_el else ""

+            href  = link_el.get_attribute("href") if link_el else ""

+            url   = normalise_url(href)

+            title_el = item.query_selector("h4.teaser__title")

+                title_el = item.query_selector("h2, h3, h4, h5, .title")

+            title = title_el.inner_text().strip() if title_el else ""

-            date_el = first_match(item, DATE_SELECTORS)

-                raw_date = date_el.get_attribute("datetime") or date_el.inner_text()

-            date = parse_date(raw_date)

+            date_el = item.query_selector("h6.teaser__date")

+            raw_date = date_el.inner_text().strip() if date_el else ""

+            date = parse_dewa_date(raw_date)

-            desc_el = item.query_selector("p, .description, .excerpt, .summary")

-            description = desc_el.inner_text().strip() if desc_el else None

+            # Description — stored in data-vl attribute of the <p>

+            desc_el = item.query_selector("p.teaser__copy")

+                    desc_el.get_attribute("data-vl")

+                    or desc_el.inner_text().strip()

-            cat_el = item.query_selector(".category, .tag, .label, [class*='categ']")

-            category = cat_el.inner_text().strip() if cat_el else None

-            print(f"[scraper] Skipping item due to error: {e}")

-    print(f"[scraper] Extracted {len(results)} press releases")

+            print(f"  [warn] Skipping item: {e}")

-def scrape_podcasts(page, debug: bool = False):

-    print(f"[scraper] Loading podcasts page → {DEWA_PODCAST_URL}")

-        page.goto(DEWA_PODCAST_URL, timeout=SCRAPER_TIMEOUT_MS, wait_until="networkidle")

-        page.wait_for_timeout(SCRAPER_WAIT_AFTER_LOAD_MS)

-        print("[scraper] Podcast page timed out — skipping")

+def scrape_press_releases(page, debug: bool = False) -> list:

+    print(f"[scraper] → News page: {DEWA_NEWS_URL}")

+    page.goto(DEWA_NEWS_URL, timeout=SCRAPER_TIMEOUT_MS, wait_until="networkidle")

+    page.wait_for_timeout(SCRAPER_WAIT_AFTER_LOAD_MS)

-        page.screenshot(path="debug_podcasts.png", full_page=True)

-        Path("debug_podcasts.html").write_text(page.content(), encoding="utf-8")

-        print("[debug] Saved debug_podcasts.png + debug_podcasts.html")

+        page.screenshot(path="debug_news.png", full_page=True)

+        Path("debug_news.html").write_text(page.content(), encoding="utf-8")

+        print("[debug] Saved debug_news.png + debug_news.html")

-        page.evaluate("window.scrollBy(0, window.innerHeight)")

+        print(f"  [scraper] Selecting year {year}…", end=" ", flush=True)

-                "button:has-text('Load more'), button:has-text('Show more'), "

-            if btn and btn.is_visible():

+            # Select the year in the dropdown

+            year_select = page.query_selector("select#form-field-years, select[name='year']")

+                print("year selector not found, skipping")

+            year_select.select_option(value=year)

+            page.wait_for_timeout(2500)   # wait for results to reload

+            # Also reset month to "All Months" to get all items in the year

+            month_select = page.query_selector("select#form-field-months, select[name='month']")

+                month_select.select_option(value="")

-        found = page.query_selector_all(sel)

-            print(f"[scraper] Podcasts using selector '{sel}' → {len(found)} items")

+            items = extract_items_from_page(page)

+            new_items = [i for i in items if i["url"] not in seen_urls]

+                seen_urls.add(i["url"])

+            all_results.extend(new_items)

+            print(f"{len(new_items)} new items ({len(items)} total on page)")

+    print(f"[scraper] Total press releases extracted: {len(all_results)}")

+# ── Podcast scraper (RSS feed) ────────────────────────────────────────────────

+def scrape_podcasts_from_rss() -> list:

+    print(f"[scraper] → Fetching RSS feed: {RSS_FEED_URL}")

+            headers={"User-Agent": "Mozilla/5.0 (compatible; DEWATracker/1.0)"}

+        with urlopen(req, timeout=30) as resp:

+        print(f"[scraper] RSS fetch error: {e}")

+        "itunes":  "http://www.itunes.com/dtds/podcast-1.0.dtd",

+        "podcast": "https://podcastindex.org/namespace/1.0",

+        "content": "http://purl.org/rss/1.0/modules/content/",

+        print(f"[scraper] XML parse error: {e}")

+    for item in channel.findall("item"):

-            title_el = first_match(item, TITLE_SELECTORS)

-            title = title_el.inner_text().strip()

-            href  = title_el.get_attribute("href") or ""

-                link_el = item.query_selector("a")

-                href = link_el.get_attribute("href") if link_el else ""

+            title = (item.findtext("title") or "").strip()

+            link  = (item.findtext("link")  or "").strip()

-            date_el = first_match(item, DATE_SELECTORS)

-            date = parse_date(raw_date)

+            # Date — RFC 822 format e.g. "Wed, 01 Apr 2026 08:58:31 GMT"

+            pub_date_raw = item.findtext("pubDate") or ""

+                    date = parsedate_to_datetime(pub_date_raw).strftime("%Y-%m-%d")

-            audio_el = item.query_selector("audio source, [src$='.mp3'], [href$='.mp3'], audio")

-                audio_url = audio_el.get_attribute("src") or audio_el.get_attribute("href")

-                audio_url = normalise_url(audio_url) if audio_url else None

+            enclosure = item.find("enclosure")

+            audio_url = enclosure.get("url") if enclosure is not None else None

-            dur_el = item.query_selector(".duration, [class*='duration'], time")

-            duration = dur_el.inner_text().strip() if dur_el else None

+            # Duration (seconds → mm:ss)

+            dur_raw  = item.findtext("itunes:duration", namespaces=ns) or ""

+            duration = seconds_to_mmss(dur_raw) if dur_raw else None

+            desc_raw = item.findtext("description") or ""

+            desc_raw = re.sub(r"<[^>]+>", "", desc_raw).strip()

+            description = desc_raw or None

+            # Language detection (Arabic title?)

+            is_arabic = bool(re.search(r'[\u0600-\u06FF]', title))

+            language  = "ar" if is_arabic else "en"

-                "audio_url": audio_url,

+                "audio_url":   audio_url,

+                "duration":    duration,

+                "description": description,

+                "language":    language,

-            print(f"[scraper] Skipping podcast item: {e}")

+            print(f"  [warn] Skipping podcast item: {e}")

-    print(f"[scraper] Extracted {len(results)} podcasts")

+    print(f"[scraper] Total podcast episodes from RSS: {len(results)}")

+# ── Match podcasts → press releases by title ──────────────────────────────────

+    """Lowercase, strip punctuation for fuzzy matching."""

-    Try to link each podcast to a press release by URL fragment or title similarity.

+    pr_id_map: { url: (id, normalised_title) }

+    1. URL slug — rss.com episode URL may contain matching text

+    2. Title substring overlap (>60% word overlap)

+    pr_titles = [(pid, ntitle) for _, (pid, ntitle) in pr_id_map.items()]

-        # Check if podcast URL matches any PR URL closely

-        for pr_url, pid in pr_id_map.items():

-            slug = pr_url.rstrip("/").split("/")[-1]

-            if slug and slug in pod["url"]:

+        pod_norm = normalise_title(pod["title"])

+        pod_words = set(pod_norm.split())

+            pr_words = set(pr_norm.split())

+            overlap = len(pod_words & pr_words) / len(pr_words)

+    linked = sum(1 for p in matched if p["pr_id"])

+    print(f"[scraper] Podcasts linked to PRs: {linked}/{len(matched)}")

-# ── Main entry point ──────────────────────────────────────────────────────────

+# ── Main ──────────────────────────────────────────────────────────────────────

-def run_scrape(debug: bool = False, headless: bool = None):

+def run_scrape(debug: bool = False, headless: bool = None) -> tuple[int, int]:

-        headless = False  # always show browser in debug mode

+        # ── 1. Press Releases ─────────────────────────────────────────────────

-            context = browser.new_context(

"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "

"AppleWebKit/537.36 (KHTML, like Gecko) "

viewport={"width": 1440, "height": 900},

-            # ── Press Releases ────────────────────────────────────────────────

pr_data = scrape_press_releases(page, debug=debug)

-                pid = db.upsert_press_release(

-                    category=pr["category"],

-                    description=pr["description"],

-                pr_id_map[pr["url"]] = pid

-            # ── Podcasts ──────────────────────────────────────────────────────

-            pod_data = scrape_podcasts(page, debug=debug)

-            pod_data = match_podcasts_to_prs(pod_data, pr_id_map)

-                    title=pod["title"],

-                    pr_id=pod.get("pr_id"),

-                    audio_url=pod.get("audio_url"),

-                    date=pod.get("date"),

-                    duration=pod.get("duration"),

+            pid = db.upsert_press_release(

+                category=pr.get("category"),

+                description=pr.get("description"),

+            pr_id_map[pr["url"]] = (pid, normalise_title(pr["title"]))

+        # ── 2. Podcasts from RSS feed ─────────────────────────────────────────

+        pod_data = scrape_podcasts_from_rss()

+        pod_data = match_podcasts_to_prs(pod_data, pr_id_map)

+                pr_id=pod.get("pr_id"),

+                audio_url=pod.get("audio_url"),

+                duration=pod.get("duration"),

db.log_scrape(prs_saved, pods_saved, status, notes)

-    print(f"\n[scraper] Done — {prs_saved} PRs, {pods_saved} podcasts saved. Status: {status}")

+    print(f"\n✅ Done — {prs_saved} PRs, {pods_saved} podcasts saved. Status: {status}")

-        print(f"[scraper] Notes: {notes}")

-    parser = argparse.ArgumentParser(description="DEWA PR Scraper")

-    parser.add_argument("--debug",   action="store_true", help="Save HTML/screenshots, open visible browser")

-    parser.add_argument("--visible", action="store_true", help="Open visible browser without debug output")

+    parser = argparse.ArgumentParser(description="DEWA PR & Podcast Scraper")

+    parser.add_argument("--debug",   action="store_true", help="Visible browser + save debug files")

+    parser.add_argument("--visible", action="store_true", help="Visible browser only")

"""
DEWA PR Tracker — Playwright-based scraper
Run directly:  python scraper.py
Debug mode:    python scraper.py --debug   (saves HTML + screenshot, opens browser)
"""
import argparse
import re
import sys
from datetime import datetime
from pathlib import Path

from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout

import database as db
from config import (
    DEWA_NEWS_URL, DEWA_PODCAST_URL, DEWA_BASE_URL,
    SCRAPER_HEADLESS, SCRAPER_TIMEOUT_MS, SCRAPER_WAIT_AFTER_LOAD_MS,
)

# ── Selector candidates (tried in order until one matches) ───────────────────
PR_ITEM_SELECTORS = [
    "article",
    ".news-item",
    ".press-release-item",
    ".listing-item",
    ".media-item",
    "[class*='news']",
    "[class*='article']",
    "[class*='press']",
    ".container li",
    ".results-list li",
]

TITLE_SELECTORS  = ["h1", "h2", "h3", "h4", ".title", "[class*='title']", "a"]
DATE_SELECTORS   = ["time", ".date", "[class*='date']", "[datetime]", "span.label"]

PODCAST_ITEM_SELECTORS = [
    ".podcast-item",
    "article",
    ".media-item",
    "[class*='podcast']",
    ".listing-item",
]

# ── Utilities ─────────────────────────────────────────────────────────────────

def normalise_url(href: str) -> str:
    if not href:
        return ""
    href = href.strip()
    if href.startswith("http"):
        return href
    if href.startswith("/"):
        return DEWA_BASE_URL + href
    return href

def parse_date(raw: str) -> str:
    """Try to parse a messy date string into YYYY-MM-DD."""
    if not raw:
        return None
    raw = raw.strip()
    formats = [
        "%d %B %Y", "%B %d, %Y", "%d/%m/%Y",
        "%Y-%m-%d", "%d-%m-%Y", "%d %b %Y",
        "%B %Y",
    ]
    for fmt in formats:
        try:
            return datetime.strptime(raw, fmt).strftime("%Y-%m-%d")
        except ValueError:
            continue
    # Fallback: grab 4-digit year
    m = re.search(r"(\d{4})", raw)
    return m.group(1) + "-01-01" if m else None

def first_match(element, selectors):
    for sel in selectors:
        try:
            el = element.query_selector(sel)
            if el:
                return el
        except Exception:
            continue
    return None

# ── Core scraping logic ───────────────────────────────────────────────────────

def scrape_press_releases(page, debug: bool = False):
    print(f"[scraper] Loading news page → {DEWA_NEWS_URL}")
    page.goto(DEWA_NEWS_URL, timeout=SCRAPER_TIMEOUT_MS, wait_until="networkidle")
    page.wait_for_timeout(SCRAPER_WAIT_AFTER_LOAD_MS)

    if debug:
        page.screenshot(path="debug_news.png", full_page=True)
        Path("debug_news.html").write_text(page.content(), encoding="utf-8")
        print("[debug] Saved debug_news.png + debug_news.html")

    # Auto-scroll to trigger lazy-loading
    for _ in range(5):
        page.evaluate("window.scrollBy(0, window.innerHeight)")
        page.wait_for_timeout(800)

    # Try "Load more" / pagination buttons
    for _ in range(20):
        try:
            btn = page.query_selector(
                "button:has-text('Load more'), button:has-text('Show more'), "
                "a:has-text('Load more'), a:has-text('Next'), .pagination .next"
            )
            if btn and btn.is_visible():
                btn.click()
                page.wait_for_timeout(2000)
            else:
                break
        except Exception:
            break

    # Find the container with news items
    items = []
    for sel in PR_ITEM_SELECTORS:
        found = page.query_selector_all(sel)
        if len(found) >= 3:
            items = found
            print(f"[scraper] Using selector '{sel}' → {len(found)} items found")
            break

    if not items:
        print("[scraper] WARNING: No items found with known selectors. Run with --debug to inspect HTML.")
        return []

    results = []
    for item in items:
        try:
            # Title + URL
            title_el = first_match(item, TITLE_SELECTORS)
            if not title_el:
                continue
            title = title_el.inner_text().strip()
            href  = title_el.get_attribute("href") or ""
            if not href:
                link_el = item.query_selector("a")
                href = link_el.get_attribute("href") if link_el else ""
            url = normalise_url(href)

            if not title or not url:
                continue

            # Date
            date_el = first_match(item, DATE_SELECTORS)
            raw_date = ""
            if date_el:
                raw_date = date_el.get_attribute("datetime") or date_el.inner_text()
            date = parse_date(raw_date)

            # Category / description
            desc_el = item.query_selector("p, .description, .excerpt, .summary")
            description = desc_el.inner_text().strip() if desc_el else None

            cat_el = item.query_selector(".category, .tag, .label, [class*='categ']")
            category = cat_el.inner_text().strip() if cat_el else None

            results.append({
                "title": title,
                "url": url,
                "date": date,
                "category": category,
                "description": description,
            })
        except Exception as e:
            print(f"[scraper] Skipping item due to error: {e}")
            continue

    print(f"[scraper] Extracted {len(results)} press releases")
    return results

def scrape_podcasts(page, debug: bool = False):
    print(f"[scraper] Loading podcasts page → {DEWA_PODCAST_URL}")
    try:
        page.goto(DEWA_PODCAST_URL, timeout=SCRAPER_TIMEOUT_MS, wait_until="networkidle")
        page.wait_for_timeout(SCRAPER_WAIT_AFTER_LOAD_MS)
    except PWTimeout:
        print("[scraper] Podcast page timed out — skipping")
        return []

    if debug:
        page.screenshot(path="debug_podcasts.png", full_page=True)
        Path("debug_podcasts.html").write_text(page.content(), encoding="utf-8")
        print("[debug] Saved debug_podcasts.png + debug_podcasts.html")

    # Auto-scroll
    for _ in range(5):
        page.evaluate("window.scrollBy(0, window.innerHeight)")
        page.wait_for_timeout(800)

    # Try "Load more"
    for _ in range(20):
        try:
            btn = page.query_selector(
                "button:has-text('Load more'), button:has-text('Show more'), "
                "a:has-text('Load more'), a:has-text('Next'), .pagination .next"
            )
            if btn and btn.is_visible():
                btn.click()
                page.wait_for_timeout(2000)
            else:
                break
        except Exception:
            break

    items = []
    for sel in PODCAST_ITEM_SELECTORS:
        found = page.query_selector_all(sel)
        if len(found) >= 2:
            items = found
            print(f"[scraper] Podcasts using selector '{sel}' → {len(found)} items")
            break

    results = []
    for item in items:
        try:
            title_el = first_match(item, TITLE_SELECTORS)
            if not title_el:
                continue
            title = title_el.inner_text().strip()
            href  = title_el.get_attribute("href") or ""
            if not href:
                link_el = item.query_selector("a")
                href = link_el.get_attribute("href") if link_el else ""
            url = normalise_url(href)

            if not title or not url:
                continue

            date_el = first_match(item, DATE_SELECTORS)
            raw_date = ""
            if date_el:
                raw_date = date_el.get_attribute("datetime") or date_el.inner_text()
            date = parse_date(raw_date)

            audio_el = item.query_selector("audio source, [src$='.mp3'], [href$='.mp3'], audio")
            audio_url = None
            if audio_el:
                audio_url = audio_el.get_attribute("src") or audio_el.get_attribute("href")
                audio_url = normalise_url(audio_url) if audio_url else None

            dur_el = item.query_selector(".duration, [class*='duration'], time")
            duration = dur_el.inner_text().strip() if dur_el else None

            results.append({
                "title": title,
                "url": url,
                "audio_url": audio_url,
                "date": date,
                "duration": duration,
            })
        except Exception as e:
            print(f"[scraper] Skipping podcast item: {e}")
            continue

    print(f"[scraper] Extracted {len(results)} podcasts")
    return results

def match_podcasts_to_prs(podcasts: list, pr_id_map: dict) -> list:
    """
    Try to link each podcast to a press release by URL fragment or title similarity.
    pr_id_map: { url: id }
    """
    matched = []
    for pod in podcasts:
        pr_id = None
        # Check if podcast URL matches any PR URL closely
        for pr_url, pid in pr_id_map.items():
            slug = pr_url.rstrip("/").split("/")[-1]
            if slug and slug in pod["url"]:
                pr_id = pid
                break
        matched.append({**pod, "pr_id": pr_id})
    return matched

# ── Main entry point ──────────────────────────────────────────────────────────

def run_scrape(debug: bool = False, headless: bool = None):
    if headless is None:
        headless = SCRAPER_HEADLESS
    if debug:
        headless = False  # always show browser in debug mode

    db.init_db()

    prs_saved = 0
    pods_saved = 0
    status = "success"
    notes = ""

    try:
        with sync_playwright() as pw:
            browser = pw.chromium.launch(
                headless=headless,
                args=["--disable-blink-features=AutomationControlled"],
            )
            context = browser.new_context(
                user_agent=(
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
                    "AppleWebKit/537.36 (KHTML, like Gecko) "
                    "Chrome/122.0.0.0 Safari/537.36"
                ),
                viewport={"width": 1440, "height": 900},
            )
            page = context.new_page()

            # ── Press Releases ────────────────────────────────────────────────
            pr_data = scrape_press_releases(page, debug=debug)
            pr_id_map = {}
            for pr in pr_data:
                pid = db.upsert_press_release(
                    title=pr["title"],
                    url=pr["url"],
                    date=pr["date"],
                    category=pr["category"],
                    description=pr["description"],
                )
                pr_id_map[pr["url"]] = pid
                prs_saved += 1

            # ── Podcasts ──────────────────────────────────────────────────────
            pod_data = scrape_podcasts(page, debug=debug)
            pod_data = match_podcasts_to_prs(pod_data, pr_id_map)
            for pod in pod_data:
                db.upsert_podcast(
                    title=pod["title"],
                    url=pod["url"],
                    pr_id=pod.get("pr_id"),
                    audio_url=pod.get("audio_url"),
                    date=pod.get("date"),
                    duration=pod.get("duration"),
                )
                pods_saved += 1

            browser.close()

    except Exception as e:
        status = "error"
        notes = str(e)
        print(f"[scraper] ERROR: {e}")

    db.log_scrape(prs_saved, pods_saved, status, notes)
    print(f"\n[scraper] Done — {prs_saved} PRs, {pods_saved} podcasts saved. Status: {status}")
    if notes:
        print(f"[scraper] Notes: {notes}")

    return prs_saved, pods_saved

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="DEWA PR Scraper")
    parser.add_argument("--debug",   action="store_true", help="Save HTML/screenshots, open visible browser")
    parser.add_argument("--visible", action="store_true", help="Open visible browser without debug output")
    args = parser.parse_args()

    run_scrape(debug=args.debug, headless=not args.visible and SCRAPER_HEADLESS)


Now I need to update the database to store the podcast language field, and clean up the old test DB:


Edit

CREATE TABLE IF NOT EXISTS podcasts (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    pr_id       INTEGER REFERENCES press_releases(id),
    title       TEXT    NOT NULL,
    url         TEXT    UNIQUE NOT NULL,
    audio_url   TEXT,
    date        TEXT,
    duration    TEXT,
    scraped_at  TEXT    NOT NULL
);

CREATE TABLE IF NOT EXISTS podcasts (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    pr_id       INTEGER REFERENCES press_releases(id),
    title       TEXT    NOT NULL,
    url         TEXT    UNIQUE NOT NULL,
    audio_url   TEXT,
    date        TEXT,
    duration    TEXT,
    language    TEXT    DEFAULT 'en',
    scraped_at  TEXT    NOT NULL
);


The file /Users/khalidbinkalban/Projects/dewa-pr-tracker/database.py has been updated successfully.

CREATE TABLE IF NOT EXISTS podcasts (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    pr_id       INTEGER REFERENCES press_releases(id),
    title       TEXT    NOT NULL,
    url         TEXT    UNIQUE NOT NULL,
    audio_url   TEXT,
    date        TEXT,
    duration    TEXT,
    language    TEXT    DEFAULT 'en',
    scraped_at  TEXT    NOT NULL
);

"""
DEWA PR Tracker — Database layer (SQLite)
"""
import sqlite3
from contextlib import contextmanager
from datetime import datetime
from config import DB_PATH

# ── Schema ────────────────────────────────────────────────────────────────────

SCHEMA = """
CREATE TABLE IF NOT EXISTS press_releases (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    title       TEXT    NOT NULL,
    url         TEXT    UNIQUE NOT NULL,
    date        TEXT,                    -- ISO-8601 (YYYY-MM-DD)
    category    TEXT,
    description TEXT,
    scraped_at  TEXT    NOT NULL
);

CREATE TABLE IF NOT EXISTS podcasts (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    pr_id       INTEGER REFERENCES press_releases(id),
    title       TEXT    NOT NULL,
    url         TEXT    UNIQUE NOT NULL,
    audio_url   TEXT,
    date        TEXT,
    duration    TEXT,
    scraped_at  TEXT    NOT NULL
);

CREATE TABLE IF NOT EXISTS scrape_log (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    ran_at      TEXT NOT NULL,
    prs_found   INTEGER DEFAULT 0,
    pods_found  INTEGER DEFAULT 0,
    status      TEXT,
    notes       TEXT
);
"""

@contextmanager
def get_conn():
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    try:
        yield conn
        conn.commit()
    finally:
        conn.close()

def init_db():
    with get_conn() as conn:
        conn.executescript(SCHEMA)

# ── Upsert helpers ────────────────────────────────────────────────────────────

def upsert_press_release(title: str, url: str, date: str = None,
                          category: str = None, description: str = None) -> int:
    """Insert or update a press release. Returns the row id."""
    now = datetime.utcnow().isoformat()
    with get_conn() as conn:
        cur = conn.execute(
            """
            INSERT INTO press_releases (title, url, date, category, description, scraped_at)
            VALUES (?, ?, ?, ?, ?, ?)
            ON CONFLICT(url) DO UPDATE SET
                title       = excluded.title,
                date        = excluded.date,
                category    = excluded.category,
                description = excluded.description,
                scraped_at  = excluded.scraped_at
            """,
            (title, url, date, category, description, now),
        )
        if cur.lastrowid:
            return cur.lastrowid
        row = conn.execute("SELECT id FROM press_releases WHERE url = ?", (url,)).fetchone()
        return row["id"]

def upsert_podcast(title: str, url: str, pr_id: int = None,
                   audio_url: str = None, date: str = None, duration: str = None) -> int:
    now = datetime.utcnow().isoformat()
    with get_conn() as conn:
        cur = conn.execute(
            """
            INSERT INTO podcasts (pr_id, title, url, audio_url, date, duration, scraped_at)
            VALUES (?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(url) DO UPDATE SET
                title     = excluded.title,
                audio_url = excluded.audio_url,
                date      = excluded.date,
                duration  = excluded.duration,
                scraped_at= excluded.scraped_at
            """,
            (pr_id, title, url, audio_url, date, duration, now),
        )
        if cur.lastrowid:
            return cur.lastrowid
        row = conn.execute("SELECT id FROM podcasts WHERE url = ?", (url,)).fetchone()
        return row["id"]

def log_scrape(prs_found: int, pods_found: int, status: str, notes: str = ""):
    with get_conn() as conn:
        conn.execute(
            "INSERT INTO scrape_log (ran_at, prs_found, pods_found, status, notes) VALUES (?,?,?,?,?)",
            (datetime.utcnow().isoformat(), prs_found, pods_found, status, notes),
        )

# ── Query helpers ─────────────────────────────────────────────────────────────

def get_all_press_releases():
    with get_conn() as conn:
        return conn.execute(
            """
            SELECT
                pr.id,
                pr.title,
                pr.url,
                pr.date,
                pr.category,
                pr.description,
                pr.scraped_at,
                COUNT(p.id) AS podcast_count
            FROM press_releases pr
            LEFT JOIN podcasts p ON p.pr_id = pr.id
            GROUP BY pr.id
            ORDER BY pr.date DESC, pr.id DESC
            """
        ).fetchall()

def get_all_podcasts():
    with get_conn() as conn:
        return conn.execute(
            """
            SELECT p.*, pr.title AS pr_title
            FROM podcasts p
            LEFT JOIN press_releases pr ON pr.id = p.pr_id
            ORDER BY p.date DESC, p.id DESC
            """
        ).fetchall()

def get_counts_by_day():
    with get_conn() as conn:
        return conn.execute(
            "SELECT date, COUNT(*) as count FROM press_releases WHERE date IS NOT NULL GROUP BY date ORDER BY date"
        ).fetchall()

def get_counts_by_month():
    with get_conn() as conn:
        return conn.execute(
            """
            SELECT substr(date,1,7) AS month, COUNT(*) AS count
            FROM press_releases WHERE date IS NOT NULL
            GROUP BY month ORDER BY month
            """
        ).fetchall()

def get_scrape_log():
    with get_conn() as conn:
        return conn.execute(
            "SELECT * FROM scrape_log ORDER BY ran_at DESC LIMIT 50"
        ).fetchall()

def get_stats():
    with get_conn() as conn:
        total_prs    = conn.execute("SELECT COUNT(*) FROM press_releases").fetchone()[0]
        total_pods   = conn.execute("SELECT COUNT(*) FROM podcasts").fetchone()[0]
        last_scraped = conn.execute("SELECT MAX(scraped_at) FROM press_releases").fetchone()[0]
        return {"total_prs": total_prs, "total_podcasts": total_pods, "last_scraped": last_scraped}

audio_url   TEXT,
date        TEXT,
duration    TEXT,
+    language    TEXT    DEFAULT 'en',
scraped_at  TEXT    NOT NULL
);


def upsert_podcast(title: str, url: str, pr_id: int = None,
                   audio_url: str = None, date: str = None, duration: str = None) -> int:
    now = datetime.utcnow().isoformat()
    with get_conn() as conn:
        cur = conn.execute(
            """
            INSERT INTO podcasts (pr_id, title, url, audio_url, date, duration, scraped_at)
            VALUES (?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(url) DO UPDATE SET
                title     = excluded.title,
                audio_url = excluded.audio_url,
                date      = excluded.date,
                duration  = excluded.duration,
                scraped_at= excluded.scraped_at
            """,
            (pr_id, title, url, audio_url, date, duration, now),
        )
        if cur.lastrowid:
            return cur.lastrowid
        row = conn.execute("SELECT id FROM podcasts WHERE url = ?", (url,)).fetchone()
        return row["id"]

def upsert_podcast(title: str, url: str, pr_id: int = None,
                   audio_url: str = None, date: str = None, duration: str = None,
                   language: str = "en") -> int:
    now = datetime.utcnow().isoformat()
    with get_conn() as conn:
        cur = conn.execute(
            """
            INSERT INTO podcasts (pr_id, title, url, audio_url, date, duration, language, scraped_at)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(url) DO UPDATE SET
                title     = excluded.title,
                audio_url = excluded.audio_url,
                date      = excluded.date,
                duration  = excluded.duration,
                language  = excluded.language,
                scraped_at= excluded.scraped_at
            """,
            (pr_id, title, url, audio_url, date, duration, language, now),
        )
        if cur.lastrowid:
            return cur.lastrowid
        row = conn.execute("SELECT id FROM podcasts WHERE url = ?", (url,)).fetchone()
        return row["id"]


def upsert_podcast(title: str, url: str, pr_id: int = None,
                   audio_url: str = None, date: str = None, duration: str = None,
                   language: str = "en") -> int:
    now = datetime.utcnow().isoformat()
    with get_conn() as conn:
        cur = conn.execute(
            """
            INSERT INTO podcasts (pr_id, title, url, audio_url, date, duration, language, scraped_at)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(url) DO UPDATE SET
                title     = excluded.title,
                audio_url = excluded.audio_url,
                date      = excluded.date,
                duration  = excluded.duration,
                language  = excluded.language,
                scraped_at= excluded.scraped_at
            """,
            (pr_id, title, url, audio_url, date, duration, language, now),
        )
        if cur.lastrowid:
            return cur.lastrowid
        row = conn.execute("SELECT id FROM podcasts WHERE url = ?", (url,)).fetchone()
        return row["id"]

"""
DEWA PR Tracker — Database layer (SQLite)
"""
import sqlite3
from contextlib import contextmanager
from datetime import datetime
from config import DB_PATH

# ── Schema ────────────────────────────────────────────────────────────────────

SCHEMA = """
CREATE TABLE IF NOT EXISTS press_releases (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    title       TEXT    NOT NULL,
    url         TEXT    UNIQUE NOT NULL,
    date        TEXT,                    -- ISO-8601 (YYYY-MM-DD)
    category    TEXT,
    description TEXT,
    scraped_at  TEXT    NOT NULL
);

CREATE TABLE IF NOT EXISTS podcasts (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    pr_id       INTEGER REFERENCES press_releases(id),
    title       TEXT    NOT NULL,
    url         TEXT    UNIQUE NOT NULL,
    audio_url   TEXT,
    date        TEXT,
    duration    TEXT,
    language    TEXT    DEFAULT 'en',
    scraped_at  TEXT    NOT NULL
);

CREATE TABLE IF NOT EXISTS scrape_log (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    ran_at      TEXT NOT NULL,
    prs_found   INTEGER DEFAULT 0,
    pods_found  INTEGER DEFAULT 0,
    status      TEXT,
    notes       TEXT
);
"""

@contextmanager
def get_conn():
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    try:
        yield conn
        conn.commit()
    finally:
        conn.close()

def init_db():
    with get_conn() as conn:
        conn.executescript(SCHEMA)

# ── Upsert helpers ────────────────────────────────────────────────────────────

def upsert_press_release(title: str, url: str, date: str = None,
                          category: str = None, description: str = None) -> int:
    """Insert or update a press release. Returns the row id."""
    now = datetime.utcnow().isoformat()
    with get_conn() as conn:
        cur = conn.execute(
            """
            INSERT INTO press_releases (title, url, date, category, description, scraped_at)
            VALUES (?, ?, ?, ?, ?, ?)
            ON CONFLICT(url) DO UPDATE SET
                title       = excluded.title,
                date        = excluded.date,
                category    = excluded.category,
                description = excluded.description,
                scraped_at  = excluded.scraped_at
            """,
            (title, url, date, category, description, now),
        )
        if cur.lastrowid:
            return cur.lastrowid
        row = conn.execute("SELECT id FROM press_releases WHERE url = ?", (url,)).fetchone()
        return row["id"]

def upsert_podcast(title: str, url: str, pr_id: int = None,
                   audio_url: str = None, date: str = None, duration: str = None) -> int:
    now = datetime.utcnow().isoformat()
    with get_conn() as conn:
        cur = conn.execute(
            """
            INSERT INTO podcasts (pr_id, title, url, audio_url, date, duration, scraped_at)
            VALUES (?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(url) DO UPDATE SET
                title     = excluded.title,
                audio_url = excluded.audio_url,
                date      = excluded.date,
                duration  = excluded.duration,
                scraped_at= excluded.scraped_at
            """,
            (pr_id, title, url, audio_url, date, duration, now),
        )
        if cur.lastrowid:
            return cur.lastrowid
        row = conn.execute("SELECT id FROM podcasts WHERE url = ?", (url,)).fetchone()
        return row["id"]

def log_scrape(prs_found: int, pods_found: int, status: str, notes: str = ""):
    with get_conn() as conn:
        conn.execute(
            "INSERT INTO scrape_log (ran_at, prs_found, pods_found, status, notes) VALUES (?,?,?,?,?)",
            (datetime.utcnow().isoformat(), prs_found, pods_found, status, notes),
        )

# ── Query helpers ─────────────────────────────────────────────────────────────

def get_all_press_releases():
    with get_conn() as conn:
        return conn.execute(
            """
            SELECT
                pr.id,
                pr.title,
                pr.url,
                pr.date,
                pr.category,
                pr.description,
                pr.scraped_at,
                COUNT(p.id) AS podcast_count
            FROM press_releases pr
            LEFT JOIN podcasts p ON p.pr_id = pr.id
            GROUP BY pr.id
            ORDER BY pr.date DESC, pr.id DESC
            """
        ).fetchall()

def get_all_podcasts():
    with get_conn() as conn:
        return conn.execute(
            """
            SELECT p.*, pr.title AS pr_title
            FROM podcasts p
            LEFT JOIN press_releases pr ON pr.id = p.pr_id
            ORDER BY p.date DESC, p.id DESC
            """
        ).fetchall()

def get_counts_by_day():
    with get_conn() as conn:
        return conn.execute(
            "SELECT date, COUNT(*) as count FROM press_releases WHERE date IS NOT NULL GROUP BY date ORDER BY date"
        ).fetchall()

def get_counts_by_month():
    with get_conn() as conn:
        return conn.execute(
            """
            SELECT substr(date,1,7) AS month, COUNT(*) AS count
            FROM press_releases WHERE date IS NOT NULL
            GROUP BY month ORDER BY month
            """
        ).fetchall()

def get_scrape_log():
    with get_conn() as conn:
        return conn.execute(
            "SELECT * FROM scrape_log ORDER BY ran_at DESC LIMIT 50"
        ).fetchall()

def get_stats():
    with get_conn() as conn:
        total_prs    = conn.execute("SELECT COUNT(*) FROM press_releases").fetchone()[0]
        total_pods   = conn.execute("SELECT COUNT(*) FROM podcasts").fetchone()[0]
        last_scraped = conn.execute("SELECT MAX(scraped_at) FROM press_releases").fetchone()[0]
        return {"total_prs": total_prs, "total_podcasts": total_pods, "last_scraped": last_scraped}

-                   audio_url: str = None, date: str = None, duration: str = None) -> int:

+                   audio_url: str = None, date: str = None, duration: str = None,

+                   language: str = "en") -> int:

-            INSERT INTO podcasts (pr_id, title, url, audio_url, date, duration, scraped_at)

+            INSERT INTO podcasts (pr_id, title, url, audio_url, date, duration, language, scraped_at)

+                language  = excluded.language,

-            (pr_id, title, url, audio_url, date, duration, now),

+            (pr_id, title, url, audio_url, date, duration, language, now),


Bash

rm -f dewa_tracker.db && source venv/bin/activate && python scraper.py 2>&1

Run full scraper with correct selectors


[scraper] → News page: https://www.dewa.gov.ae/en/about-us/media-publications/latest-news
  [scraper] Selecting year 2019… 9 new items (9 total on page)
  [scraper] Selecting year 2020… 9 new items (9 total on page)
  [scraper] Selecting year 2021… 9 new items (9 total on page)
  [scraper] Selecting year 2022… 9 new items (9 total on page)
  [scraper] Selecting year 2023… 9 new items (9 total on page)
  [scraper] Selecting year 2024… 9 new items (9 total on page)
  [scraper] Selecting year 2025… 9 new items (9 total on page)
  [scraper] Selecting year 2026… 9 new items (9 total on page)
[scraper] Total press releases extracted: 72
[scraper] → Fetching RSS feed: https://media.rss.com/dewa-news/feed.xml
[scraper] Total podcast episodes from RSS: 691
[scraper] Podcasts linked to PRs: 67/691

✅ Done — 72 PRs, 691 podcasts saved. Status: success


The scraper works but only gets 9 items per year — the page has pagination. Let me intercept the Sitecore API calls to find the real pagination mechanism:


source venv/bin/activate && python3 -c "
from playwright.sync_api import sync_playwright
from pathlib import Path

api_calls = []

with sync_playwright() as pw:
    browser = pw.chromium.launch(headless=True)
    ctx = browser.new_context(user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36')
    page = ctx.new_page()

    # Intercept network requests
    def on_request(req):
        if 'ListingV1' in req.url or 'listing' in req.url.lower() or 'api' in req.url and 'sitecore' in req.url:
            print(f'REQUEST: {req.method} {req.url}')
            if req.method == 'POST':
                print(f'  POST data: {req.post_data}')

    def on_response(resp):
        if 'ListingV1' in resp.url:
            print(f'RESPONSE: {resp.status} {resp.url}')
            try:
                body = resp.body()
                Path('debug_listingv1_response.html').write_bytes(body)
                print(f'  Body saved ({len(body)} bytes)')
            except Exception as e:
                print(f'  Body error: {e}')

    page.on('request', on_request)
    page.on('response', on_response)

    page.goto('https://www.dewa.gov.ae/en/about-us/media-publications/latest-news', wait_until='networkidle', timeout=30000)
    page.wait_for_timeout(3000)

    # Select year 2025 and observe the API call
    year_select = page.query_selector('select#form-field-years, select[name=\"year\"]')
    year_select.select_option(value='2025')
    page.wait_for_timeout(3000)

    browser.close()
" 2>&1

Intercept Sitecore API calls when year is selected


REQUEST: GET https://www.dewa.gov.ae/Scripts/External/nml/listing.js?v=9
REQUEST: POST https://www.dewa.gov.ae/api/sitecore/Renderings/ListingV1
  POST data: itemId=d4a4a0bb-5c23-4b19-ad15-e2ff9c86a9ff&month=3&year=2026
RESPONSE: 200 https://www.dewa.gov.ae/api/sitecore/Renderings/ListingV1
  Body saved (16525 bytes)
REQUEST: POST https://www.dewa.gov.ae/api/sitecore/Renderings/ListingV1
  POST data: itemId=d4a4a0bb-5c23-4b19-ad15-e2ff9c86a9ff&month=3&year=2025
RESPONSE: 200 https://www.dewa.gov.ae/api/sitecore/Renderings/ListingV1
  Body saved (18393 bytes)


python3 -c "
from pathlib import Path
import re
html = Path('debug_listingv1_response.html').read_bytes().decode('utf-8', errors='ignore')
print('Size:', len(html))

# Count items
items = html.count('m9-teaser--news_all')
print('Items in response:', items)

# Check for pagination/total count
for kw in ['total', 'count', 'page', 'more', 'next', 'pagination']:
    idx = html.lower().find(kw)
    if idx >= 0:
        snippet = html[max(0,idx-50):idx+150]
        print(f'{kw}: {snippet[:200]}')
        print('---')
"

Analyze ListingV1 API response for pagination


Size: 18340
Items in response: 9
count: ity and Water Authority (DEWA), met Fakher Bader, Country Chairman of the Shell Group of Companies in Iraq and the UAE, to discuss measures to strengthen partnerships in renewable energy, innovation a
---
page: on>
    </div>

    <input type="hidden" name="page" value="1" id="page-value" />
</div>

---
more: ullwidth" data-submission-text=" Loading" id="loadmore">More</button>
    </div>

    <input type="hidden" name="page" value="1" id="page-value" />
</div>

---


There's a "More" button and a `page` hidden input — it's paginated! I can hit the API directly with `requests`. Let me test:


source venv/bin/activate && pip install -q requests && python3 -c "
import requests

resp = requests.post(
    'https://www.dewa.gov.ae/api/sitecore/Renderings/ListingV1',
    data={
        'itemId': 'd4a4a0bb-5c23-4b19-ad15-e2ff9c86a9ff',
        'month': '',
        'year': '2025',
        'page': '1',
    },
    headers={
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
        'Referer': 'https://www.dewa.gov.ae/en/about-us/media-publications/latest-news',
        'Origin': 'https://www.dewa.gov.ae',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'X-Requested-With': 'XMLHttpRequest',
    },
    timeout=30,
)
print('Status:', resp.status_code)
print('Size:', len(resp.text))

import re
items = resp.text.count('m9-teaser--news_all')
has_more = 'loadmore' in resp.text
print('Items:', items, '| Has more button:', has_more)

# Check page hidden input
m = re.search(r'name=\"page\" value=\"(\d+)\"', resp.text)
print('Page value in response:', m.group(1) if m else 'not found')
"

Test direct Sitecore API POST request


WARNING: You are using pip version 21.2.4; however, version 26.0.1 is available.
You should consider upgrading via the '/Users/khalidbinkalban/Projects/dewa-pr-tracker/venv/bin/python3 -m pip install --upgrade pip' command.
/Users/khalidbinkalban/Projects/dewa-pr-tracker/venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
  warnings.warn(
Status: 403
Size: 418
Items: 0 | Has more button: False
Page value in response: not found


source venv/bin/activate && python3 -c "
from playwright.sync_api import sync_playwright
from pathlib import Path

api_calls = []

with sync_playwright() as pw:
    browser = pw.chromium.launch(headless=True)
    ctx = browser.new_context(user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36')
    page = ctx.new_page()

    # Capture all ListingV1 responses
    responses = []
    def on_response(resp):
        if 'ListingV1' in resp.url:
            body = resp.body()
            import re
            items = body.count(b'm9-teaser--news_all')
            has_more = b'loadmore' in body
            post_data = ''
            responses.append({'items': items, 'has_more': has_more})
            print(f'ListingV1 response: {items} items, has_more={has_more}')

    page.on('response', on_response)

    page.goto('https://www.dewa.gov.ae/en/about-us/media-publications/latest-news', wait_until='networkidle', timeout=30000)
    page.wait_for_timeout(3000)

    # Try: reset month first, then select year 2025
    month_select = page.query_selector('select[name=\"month\"]')
    month_select.select_option(value='')
    page.wait_for_timeout(3000)

    year_select = page.query_selector('select[name=\"year\"]')
    year_select.select_option(value='2025')
    page.wait_for_timeout(3000)

    # Now count items on page
    items_on_page = page.query_selector_all('.m9-teaser--news_all')
    print(f'Items on page after year select: {len(items_on_page)}')

    # Check for More button
    more_btn = page.query_selector('#loadmore')
    print(f'More button visible: {more_btn and more_btn.is_visible()}')

    browser.close()
" 2>&1

Test year+month selection order and More button detection


ListingV1 response: 8 items, has_more=False
ListingV1 response: 9 items, has_more=True
ListingV1 response: 9 items, has_more=True
Items on page after year select: 9
More button visible: True


Now I know exactly how it works. Let me update the scraper to properly iterate all pages per year:


Edit

def scrape_press_releases(page, debug: bool = False) -> list:
    print(f"[scraper] → News page: {DEWA_NEWS_URL}")
    page.goto(DEWA_NEWS_URL, timeout=SCRAPER_TIMEOUT_MS, wait_until="networkidle")
    page.wait_for_timeout(SCRAPER_WAIT_AFTER_LOAD_MS)

    if debug:
        page.screenshot(path="debug_news.png", full_page=True)
        Path("debug_news.html").write_text(page.content(), encoding="utf-8")
        print("[debug] Saved debug_news.png + debug_news.html")

    all_results = []
    seen_urls   = set()

    for year in SCRAPE_YEARS:
        print(f"  [scraper] Selecting year {year}…", end=" ", flush=True)
        try:
            # Select the year in the dropdown
            year_select = page.query_selector("select#form-field-years, select[name='year']")
            if not year_select:
                print("year selector not found, skipping")
                continue
            year_select.select_option(value=year)
            page.wait_for_timeout(2500)   # wait for results to reload

            # Also reset month to "All Months" to get all items in the year
            month_select = page.query_selector("select#form-field-months, select[name='month']")
            if month_select:
                month_select.select_option(value="")
                page.wait_for_timeout(2000)

            items = extract_items_from_page(page)
            new_items = [i for i in items if i["url"] not in seen_urls]
            for i in new_items:
                seen_urls.add(i["url"])
            all_results.extend(new_items)
            print(f"{len(new_items)} new items ({len(items)} total on page)")

        except Exception as e:
            print(f"error: {e}")
            continue

    print(f"[scraper] Total press releases extracted: {len(all_results)}")
    return all_results

def scrape_press_releases(page, debug: bool = False) -> list:
    print(f"[scraper] → News page: {DEWA_NEWS_URL}")
    page.goto(DEWA_NEWS_URL, timeout=SCRAPER_TIMEOUT_MS, wait_until="networkidle")
    page.wait_for_timeout(SCRAPER_WAIT_AFTER_LOAD_MS)

    if debug:
        page.screenshot(path="debug_news.png", full_page=True)
        Path("debug_news.html").write_text(page.content(), encoding="utf-8")
        print("[debug] Saved debug_news.png + debug_news.html")

    all_results = []
    seen_urls   = set()

    month_select = page.query_selector("select[name='month']")
    year_select  = page.query_selector("select[name='year']")

    if not year_select:
        print("[scraper] WARNING: year selector not found — extracting current page only")
        return extract_items_from_page(page)

    for year in SCRAPE_YEARS:
        print(f"  [scraper] Year {year}:", end=" ", flush=True)
        try:
            # IMPORTANT: reset month first (fires API call), then select year
            if month_select:
                month_select.select_option(value="")
                page.wait_for_timeout(2000)
            year_select.select_option(value=year)
            page.wait_for_timeout(2500)

            # Click "More" until it disappears — loads all pages for this year
            pages_loaded = 1
            while True:
                more_btn = page.query_selector("#loadmore")
                if more_btn and more_btn.is_visible():
                    try:
                        more_btn.scroll_into_view_if_needed()
                        more_btn.click()
                        page.wait_for_timeout(2500)
                        pages_loaded += 1
                    except Exception:
                        break
                else:
                    break

            items     = extract_items_from_page(page)
            new_items = [i for i in items if i["url"] not in seen_urls]
            for i in new_items:
                seen_urls.add(i["url"])
            all_results.extend(new_items)
            print(f"{len(new_items)} items ({pages_loaded} page(s) loaded)")

        except Exception as e:
            print(f"error: {e}")
            continue

    print(f"[scraper] Total press releases extracted: {len(all_results)}")
    return all_results