def scrape_press_releases(page, debug: bool = False) -> list:
    print(f"[scraper] → News page: {DEWA_NEWS_URL}")
    page.goto(DEWA_NEWS_URL, timeout=SCRAPER_TIMEOUT_MS, wait_until="networkidle")
    page.wait_for_timeout(SCRAPER_WAIT_AFTER_LOAD_MS)

    if debug:
        page.screenshot(path="debug_news.png", full_page=True)
        Path("debug_news.html").write_text(page.content(), encoding="utf-8")
        print("[debug] Saved debug_news.png + debug_news.html")

    all_results = []
    seen_urls   = set()

    month_select = page.query_selector("select[name='month']")
    year_select  = page.query_selector("select[name='year']")

    if not year_select:
        print("[scraper] WARNING: year selector not found — extracting current page only")
        return extract_items_from_page(page)

    for year in SCRAPE_YEARS:
        print(f"  [scraper] Year {year}:", end=" ", flush=True)
        try:
            # IMPORTANT: reset month first (fires API call), then select year
            if month_select:
                month_select.select_option(value="")
                page.wait_for_timeout(2000)
            year_select.select_option(value=year)
            page.wait_for_timeout(2500)

            # Click "More" until it disappears — loads all pages for this year
            pages_loaded = 1
            while True:
                more_btn = page.query_selector("#loadmore")
                if more_btn and more_btn.is_visible():
                    try:
                        more_btn.scroll_into_view_if_needed()
                        more_btn.click()
                        page.wait_for_timeout(2500)
                        pages_loaded += 1
                    except Exception:
                        break
                else:
                    break

            items     = extract_items_from_page(page)
            new_items = [i for i in items if i["url"] not in seen_urls]
            for i in new_items:
                seen_urls.add(i["url"])
            all_results.extend(new_items)
            print(f"{len(new_items)} items ({pages_loaded} page(s) loaded)")

        except Exception as e:
            print(f"error: {e}")
            continue

    print(f"[scraper] Total press releases extracted: {len(all_results)}")
    return all_results

"""
DEWA PR Tracker — Scraper
  - Press Releases: Playwright (Chromium) — iterates all years via year filter
  - Podcasts:       Direct RSS feed parse from media.rss.com (no browser needed)

Usage:
  python scraper.py              # normal run
  python scraper.py --debug      # visible browser + saves debug HTML/screenshots
  python scraper.py --visible    # visible browser only
"""
import argparse
import re
import xml.etree.ElementTree as ET
from datetime import datetime
from email.utils import parsedate_to_datetime
from pathlib import Path
from urllib.request import urlopen, Request

from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout

import database as db
from config import (
    DEWA_NEWS_URL, DEWA_BASE_URL,
    SCRAPER_HEADLESS, SCRAPER_TIMEOUT_MS, SCRAPER_WAIT_AFTER_LOAD_MS,
)

RSS_FEED_URL = "https://media.rss.com/dewa-news/feed.xml"

# Years available on the DEWA news page
SCRAPE_YEARS = ["2019", "2020", "2021", "2022", "2023", "2024", "2025", "2026"]

# ── Utilities ─────────────────────────────────────────────────────────────────

def normalise_url(href: str) -> str:
    if not href:
        return ""
    href = href.strip()
    if href.startswith("http"):
        return href
    if href.startswith("/"):
        return DEWA_BASE_URL + href
    return href

def parse_dewa_date(raw: str) -> str:
    """Parse DEWA date format '30 March 2026' → '2026-03-30'."""
    if not raw:
        return None
    raw = raw.strip()
    try:
        return datetime.strptime(raw, "%d %B %Y").strftime("%Y-%m-%d")
    except ValueError:
        pass
    try:
        return datetime.strptime(raw, "%d %b %Y").strftime("%Y-%m-%d")
    except ValueError:
        pass
    m = re.search(r"(\d{4})", raw)
    return m.group(1) + "-01-01" if m else None

def seconds_to_mmss(secs) -> str:
    try:
        s = int(secs)
        return f"{s // 60}:{s % 60:02d}"
    except (TypeError, ValueError):
        return str(secs) if secs else None

# ── Press Release scraper (Playwright) ───────────────────────────────────────

def extract_items_from_page(page) -> list:
    """Extract all .m9-teaser--news_all items currently visible on the page."""
    items = page.query_selector_all(".m9-teaser--news_all")
    results = []
    for item in items:
        try:
            # Link — the whole card is an
            link_el = item.query_selector("a.teaser__link")
            if not link_el:
                link_el = item.query_selector("a")
            href  = link_el.get_attribute("href") if link_el else ""
            url   = normalise_url(href)

            # Title
            title_el = item.query_selector("h4.teaser__title")
            if not title_el:
                title_el = item.query_selector("h2, h3, h4, h5, .title")
            title = title_el.inner_text().strip() if title_el else ""

            if not title or not url:
                continue

            # Date — "30 March 2026"
            date_el = item.query_selector("h6.teaser__date")
            raw_date = date_el.inner_text().strip() if date_el else ""
            date = parse_dewa_date(raw_date)

            # Description — stored in data-vl attribute of the

            desc_el = item.query_selector("p.teaser__copy")
            description = None
            if desc_el:
                description = (
                    desc_el.get_attribute("data-vl")
                    or desc_el.inner_text().strip()
                    or None
                )

            results.append({
                "title": title,
                "url": url,
                "date": date,
                "description": description,
                "category": None,
            })
        except Exception as e:
            print(f"  [warn] Skipping item: {e}")
    return results

def scrape_press_releases(page, debug: bool = False) -> list:
    print(f"[scraper] → News page: {DEWA_NEWS_URL}")
    page.goto(DEWA_NEWS_URL, timeout=SCRAPER_TIMEOUT_MS, wait_until="networkidle")
    page.wait_for_timeout(SCRAPER_WAIT_AFTER_LOAD_MS)

    if debug:
        page.screenshot(path="debug_news.png", full_page=True)
        Path("debug_news.html").write_text(page.content(), encoding="utf-8")
        print("[debug] Saved debug_news.png + debug_news.html")

    all_results = []
    seen_urls   = set()

    for year in SCRAPE_YEARS:
        print(f"  [scraper] Selecting year {year}…", end=" ", flush=True)
        try:
            # Select the year in the dropdown
            year_select = page.query_selector("select#form-field-years, select[name='year']")
            if not year_select:
                print("year selector not found, skipping")
                continue
            year_select.select_option(value=year)
            page.wait_for_timeout(2500)   # wait for results to reload

            # Also reset month to "All Months" to get all items in the year
            month_select = page.query_selector("select#form-field-months, select[name='month']")
            if month_select:
                month_select.select_option(value="")
                page.wait_for_timeout(2000)

            items = extract_items_from_page(page)
            new_items = [i for i in items if i["url"] not in seen_urls]
            for i in new_items:
                seen_urls.add(i["url"])
            all_results.extend(new_items)
            print(f"{len(new_items)} new items ({len(items)} total on page)")

        except Exception as e:
            print(f"error: {e}")
            continue

    print(f"[scraper] Total press releases extracted: {len(all_results)}")
    return all_results

# ── Podcast scraper (RSS feed) ────────────────────────────────────────────────

def scrape_podcasts_from_rss() -> list:
    print(f"[scraper] → Fetching RSS feed: {RSS_FEED_URL}")
    try:
        req = Request(
            RSS_FEED_URL,
            headers={"User-Agent": "Mozilla/5.0 (compatible; DEWATracker/1.0)"}
        )
        with urlopen(req, timeout=30) as resp:
            raw_xml = resp.read()
    except Exception as e:
        print(f"[scraper] RSS fetch error: {e}")
        return []

    # Parse XML
    ns = {
        "itunes":  "http://www.itunes.com/dtds/podcast-1.0.dtd",
        "podcast": "https://podcastindex.org/namespace/1.0",
        "content": "http://purl.org/rss/1.0/modules/content/",
    }

    try:
        root = ET.fromstring(raw_xml)
    except ET.ParseError as e:
        print(f"[scraper] XML parse error: {e}")
        return []

    channel = root.find("channel")
    if not channel:
        return []

    results = []
    for item in channel.findall("item"):
        try:
            title = (item.findtext("title") or "").strip()
            link  = (item.findtext("link")  or "").strip()

            if not title or not link:
                continue

            # Date — RFC 822 format e.g. "Wed, 01 Apr 2026 08:58:31 GMT"
            pub_date_raw = item.findtext("pubDate") or ""
            date = None
            if pub_date_raw:
                try:
                    date = parsedate_to_datetime(pub_date_raw).strftime("%Y-%m-%d")
                except Exception:
                    pass

            # MP3 URL from <enclosure>
            enclosure = item.find("enclosure")
            audio_url = enclosure.get("url") if enclosure is not None else None

            # Duration (seconds → mm:ss)
            dur_raw  = item.findtext("itunes:duration", namespaces=ns) or ""
            duration = seconds_to_mmss(dur_raw) if dur_raw else None

            # Description
            desc_raw = item.findtext("description") or ""
            desc_raw = re.sub(r"<[^>]+>", "", desc_raw).strip()
            description = desc_raw or None

            # Language detection (Arabic title?)
            is_arabic = bool(re.search(r'[\u0600-\u06FF]', title))
            language  = "ar" if is_arabic else "en"

            results.append({
                "title":       title,
                "url":         link,
                "audio_url":   audio_url,
                "date":        date,
                "duration":    duration,
                "description": description,
                "language":    language,
            })
        except Exception as e:
            print(f"  [warn] Skipping podcast item: {e}")
            continue

    print(f"[scraper] Total podcast episodes from RSS: {len(results)}")
    return results

# ── Match podcasts → press releases by title ──────────────────────────────────

def normalise_title(t: str) -> str:
    """Lowercase, strip punctuation for fuzzy matching."""
    t = t.lower()
    t = re.sub(r"[^\w\s]", " ", t)
    t = re.sub(r"\s+", " ", t).strip()
    return t

def match_podcasts_to_prs(podcasts: list, pr_id_map: dict) -> list:
    """
    pr_id_map: { url: (id, normalised_title) }
    Match by:
    1. URL slug — rss.com episode URL may contain matching text
    2. Title substring overlap (>60% word overlap)
    """
    pr_titles = [(pid, ntitle) for _, (pid, ntitle) in pr_id_map.items()]

    matched = []
    for pod in podcasts:
        pr_id = None
        pod_norm = normalise_title(pod["title"])
        pod_words = set(pod_norm.split())

        best_score = 0
        best_pid   = None
        for pid, pr_norm in pr_titles:
            pr_words = set(pr_norm.split())
            if not pr_words:
                continue
            overlap = len(pod_words & pr_words) / len(pr_words)
            if overlap > best_score:
                best_score = overlap
                best_pid   = pid

        if best_score >= 0.5:
            pr_id = best_pid

        matched.append({**pod, "pr_id": pr_id})

    linked = sum(1 for p in matched if p["pr_id"])
    print(f"[scraper] Podcasts linked to PRs: {linked}/{len(matched)}")
    return matched

# ── Main ──────────────────────────────────────────────────────────────────────

def run_scrape(debug: bool = False, headless: bool = None) -> tuple[int, int]:
    if headless is None:
        headless = SCRAPER_HEADLESS
    if debug:
        headless = False

    db.init_db()

    prs_saved  = 0
    pods_saved = 0
    status     = "success"
    notes      = ""

    try:
        # ── 1. Press Releases ─────────────────────────────────────────────────
        with sync_playwright() as pw:
            browser = pw.chromium.launch(
                headless=headless,
                args=["--disable-blink-features=AutomationControlled"],
            )
            ctx = browser.new_context(
                user_agent=(
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
                    "AppleWebKit/537.36 (KHTML, like Gecko) "
                    "Chrome/122.0.0.0 Safari/537.36"
                ),
                viewport={"width": 1440, "height": 900},
            )
            page = ctx.new_page()
            pr_data = scrape_press_releases(page, debug=debug)
            browser.close()

        pr_id_map = {}
        for pr in pr_data:
            pid = db.upsert_press_release(
                title=pr["title"],
                url=pr["url"],
                date=pr["date"],
                category=pr.get("category"),
                description=pr.get("description"),
            )
            pr_id_map[pr["url"]] = (pid, normalise_title(pr["title"]))
            prs_saved += 1

        # ── 2. Podcasts from RSS feed ─────────────────────────────────────────
        pod_data = scrape_podcasts_from_rss()
        pod_data = match_podcasts_to_prs(pod_data, pr_id_map)
        for pod in pod_data:
            db.upsert_podcast(
                title=pod["title"],
                url=pod["url"],
                pr_id=pod.get("pr_id"),
                audio_url=pod.get("audio_url"),
                date=pod.get("date"),
                duration=pod.get("duration"),
            )
            pods_saved += 1

    except Exception as e:
        status = "error"
        notes  = str(e)
        print(f"[scraper] ERROR: {e}")

    db.log_scrape(prs_saved, pods_saved, status, notes)
    print(f"\n✅ Done — {prs_saved} PRs, {pods_saved} podcasts saved. Status: {status}")
    if notes:
        print(f"   Notes: {notes}")

    return prs_saved, pods_saved

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="DEWA PR & Podcast Scraper")
    parser.add_argument("--debug",   action="store_true", help="Visible browser + save debug files")
    parser.add_argument("--visible", action="store_true", help="Visible browser only")
    args = parser.parse_args()

    run_scrape(debug=args.debug, headless=not args.visible and SCRAPER_HEADLESS)

+    month_select = page.query_selector("select[name='month']")

+    year_select  = page.query_selector("select[name='year']")

+        print("[scraper] WARNING: year selector not found — extracting current page only")

+        return extract_items_from_page(page)

-        print(f"  [scraper] Selecting year {year}…", end=" ", flush=True)

+        print(f"  [scraper] Year {year}:", end=" ", flush=True)

-            # Select the year in the dropdown

-            year_select = page.query_selector("select#form-field-years, select[name='year']")

-                print("year selector not found, skipping")

-            year_select.select_option(value=year)

-            page.wait_for_timeout(2500)   # wait for results to reload

-            # Also reset month to "All Months" to get all items in the year

-            month_select = page.query_selector("select#form-field-months, select[name='month']")

+            # IMPORTANT: reset month first (fires API call), then select year

+            year_select.select_option(value=year)

+            page.wait_for_timeout(2500)

-            items = extract_items_from_page(page)

+            # Click "More" until it disappears — loads all pages for this year

+                more_btn = page.query_selector("#loadmore")

+                if more_btn and more_btn.is_visible():

+                        more_btn.scroll_into_view_if_needed()

+                        more_btn.click()

+                        page.wait_for_timeout(2500)

+                        pages_loaded += 1

+            items     = extract_items_from_page(page)

-            print(f"{len(new_items)} new items ({len(items)} total on page)")

+            print(f"{len(new_items)} items ({pages_loaded} page(s) loaded)")


Bash

rm -f dewa_tracker.db && source venv/bin/activate && python scraper.py 2>&1

Run full scraper with pagination support


[scraper] → News page: https://www.dewa.gov.ae/en/about-us/media-publications/latest-news
  [scraper] Year 2019: 135 items (15 page(s) loaded)
  [scraper] Year 2020: 99 items (11 page(s) loaded)
  [scraper] Year 2021: 144 items (16 page(s) loaded)
  [scraper] Year 2022: 117 items (13 page(s) loaded)
  [scraper] Year 2023: 36 items (4 page(s) loaded)
  [scraper] Year 2024: 90 items (10 page(s) loaded)
  [scraper] Year 2025: 426 items (48 page(s) loaded)
  [scraper] Year 2026: 71 items (8 page(s) loaded)
[scraper] Total press releases extracted: 1118
[scraper] → Fetching RSS feed: https://media.rss.com/dewa-news/feed.xml
[scraper] Total podcast episodes from RSS: 691
[scraper] Podcasts linked to PRs: 342/691

✅ Done — 1118 PRs, 691 podcasts saved. Status: success


**1,118 press releases and 691 podcasts** — all in the database. Now let's launch the dashboard:


source venv/bin/activate && python3 -c "
import database as db
stats = db.get_stats()
print('Stats:', dict(stats))
prs = db.get_all_press_releases()
print('Sample PRs:')
for p in list(prs)[:3]:
    print(' ', dict(p))
pods = db.get_all_podcasts()
print('Sample podcasts:')
for p in list(pods)[:3]:
    print(' ', dict(p))
"

Verify database contents


Stats: {'total_prs': 1117, 'total_podcasts': 691, 'last_scraped': '2026-04-04T04:46:55.158704'}
Sample PRs:
  {'id': 1048, 'title': 'DEWA supports Dubai Autism Center’s 20th Autism Awareness Campaign as Sustainability Partner', 'url': 'https://www.dewa.gov.ae/en/about-us/media-publications/latest-news/2026/3/dewa-supports-dubai-autism', 'date': '2026-03-30', 'category': None, 'description': 'As a part of its social responsibility, Dubai Electricity and Water Authority (DEWA) is supporting the Dubai Autism Centre’s 20th Autism Awareness Campaign as Sustainability Partner. The annual campaign is held under the patronage of HH Sheikh Hamdan bin Mohammed bin Rashid Al Maktoum, Crown Prince of Dubai, Deputy Prime Minister and Minister of Defence of the UAE, and Chairman of The Executive Council of Dubai. It aims to move beyond the concept of ‘formal accessibility’ to achieve true environmental adaptation, meeting the individual needs of every person with autism through a dynamic and responsive approach. ', 'scraped_at': '2026-04-04T04:46:55.137891', 'podcast_count': 1}
  {'id': 1049, 'title': "Empower Annual General Meeting approves cash dividends of D 437.5 million supported by the company's strong performance", 'url': 'https://www.dewa.gov.ae/en/about-us/media-publications/latest-news/2026/3/empower-annual-general-meeting', 'date': '2026-03-26', 'category': None, 'description': 'The Annual General Meeting (AGM) of the Emirates Central Cooling Systems Corporation PJSC (Empower), the world’s largest district cooling services provider, held with a quorum of 85% of the company’s paid-up share capital, approved the Board of Directors’ proposal to distribute cash dividends of AED 437.5 million (4.375 fils per share equivalent to 43.75% of the company’s paid-up capital) to shareholders for second half of the year 2025.', 'scraped_at': '2026-04-04T04:46:55.138142', 'podcast_count': 1}
  {'id': 1050, 'title': 'DEWA’s Damj initiative supports empowerment of People of Determination in the workplace and wider community', 'url': 'https://www.dewa.gov.ae/en/about-us/media-publications/latest-news/2026/3/dewas-damj-initiative-supports-empowerment', 'date': '2026-03-22', 'category': None, 'description': 'Dubai Electricity and Water Authority (DEWA) has reinforced its commitment to empowering People of Determination in the workplace and wider community through its Damj (Arabic for inclusion) initiative, with employee happiness among People of Determination reaching 98.02% in 2025, according to the Dubai Government Excellence Programme. Damj provides targeted support to employees, including those caring for children who are People of Determination, by addressing their educational needs, offering specialised resources and ensuring access to equipment that enables full participation and productivity.', 'scraped_at': '2026-04-04T04:46:55.138407', 'podcast_count': 1}
Sample podcasts:
  {'id': 2, 'pr_id': None, 'title': 'تحت رعاية سمو الشيخ حمدان بن محمد بن راشد آل مكتوم هيئة كهرباء ومياه دبي شريك الاستدامة للحملة السنوية العشرين للتوعية بالتوحد', 'url': 'https://rss.com/podcasts/dewa-news/2679700', 'audio_url': 'https://content.rss.com/episodes/331008/2679700/dewa-news/2026_04_01_06_52_58_b2751f61-91c6-44ff-941d-0bfa9ab0928a.mp3', 'date': '2026-04-01', 'duration': '3:22', 'language': 'en', 'scraped_at': '2026-04-04T04:46:56.898309', 'pr_title': None}
  {'id': 1, 'pr_id': 1048, 'title': 'Campaign held under the patronage of HH Sheikh Hamdan bin Mohammed bin Rashid Al Maktoum DEWA supports Dubai Autism Center’s 20th Autism Awareness Campaign as Sustainability Partner', 'url': 'https://rss.com/podcasts/dewa-news/2679905', 'audio_url': 'https://content.rss.com/episodes/331008/2679905/dewa-news/2026_04_01_08_57_46_9e4716e2-4221-4269-9b02-a9f6cb3705f8.mp3', 'date': '2026-04-01', 'duration': '2:39', 'language': 'en', 'scraped_at': '2026-04-04T04:46:56.897464', 'pr_title': 'DEWA supports Dubai Autism Center’s 20th Autism Awareness Campaign as Sustainability Partner'}
  {'id': 4, 'pr_id': 1049, 'title': "Empower Annual General Meeting approves cash dividends of AED 437.5 million supported by the company's strong performance", 'url': 'https://rss.com/podcasts/dewa-news/2663615', 'audio_url': 'https://content.rss.com/episodes/331008/2663615/dewa-news/2026_03_27_05_38_21_1d5ef8f9-f4ea-4ceb-a94b-e8fe17a41e7c.mp3', 'date': '2026-03-27', 'duration': '3:05', 'language': 'en', 'scraped_at': '2026-04-04T04:46:56.899040', 'pr_title': "Empower Annual General Meeting approves cash dividends of D 437.5 million supported by the company's strong performance"}


Data looks great. Now let me also fix the language detection for Arabic podcasts in the database, then launch the dashboard:


Edit

def get_all_podcasts():
    with get_conn() as conn:
        return conn.execute(
            """
            SELECT p.*, pr.title AS pr_title
            FROM podcasts p
            LEFT JOIN press_releases pr ON pr.id = p.pr_id
            ORDER BY p.date DESC, p.id DESC
            """
        ).fetchall()

def get_all_podcasts(language: str = None):
    with get_conn() as conn:
        if language:
            return conn.execute(
                """
                SELECT p.*, pr.title AS pr_title
                FROM podcasts p
                LEFT JOIN press_releases pr ON pr.id = p.pr_id
                WHERE p.language = ?
                ORDER BY p.date DESC, p.id DESC
                """,
                (language,),
            ).fetchall()
        return conn.execute(
            """
            SELECT p.*, pr.title AS pr_title
            FROM podcasts p
            LEFT JOIN press_releases pr ON pr.id = p.pr_id
            ORDER BY p.date DESC, p.id DESC
            """
        ).fetchall()


The file /Users/khalidbinkalban/Projects/dewa-pr-tracker/database.py has been updated successfully.

def get_all_podcasts(language: str = None):
    with get_conn() as conn:
        if language:
            return conn.execute(
                """
                SELECT p.*, pr.title AS pr_title
                FROM podcasts p
                LEFT JOIN press_releases pr ON pr.id = p.pr_id
                WHERE p.language = ?
                ORDER BY p.date DESC, p.id DESC
                """,
                (language,),
            ).fetchall()
        return conn.execute(
            """
            SELECT p.*, pr.title AS pr_title
            FROM podcasts p
            LEFT JOIN press_releases pr ON pr.id = p.pr_id
            ORDER BY p.date DESC, p.id DESC
            """
        ).fetchall()

"""
DEWA PR Tracker — Database layer (SQLite)
"""
import sqlite3
from contextlib import contextmanager
from datetime import datetime
from config import DB_PATH

# ── Schema ────────────────────────────────────────────────────────────────────

SCHEMA = """
CREATE TABLE IF NOT EXISTS press_releases (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    title       TEXT    NOT NULL,
    url         TEXT    UNIQUE NOT NULL,
    date        TEXT,                    -- ISO-8601 (YYYY-MM-DD)
    category    TEXT,
    description TEXT,
    scraped_at  TEXT    NOT NULL
);

CREATE TABLE IF NOT EXISTS podcasts (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    pr_id       INTEGER REFERENCES press_releases(id),
    title       TEXT    NOT NULL,
    url         TEXT    UNIQUE NOT NULL,
    audio_url   TEXT,
    date        TEXT,
    duration    TEXT,
    language    TEXT    DEFAULT 'en',
    scraped_at  TEXT    NOT NULL
);

CREATE TABLE IF NOT EXISTS scrape_log (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    ran_at      TEXT NOT NULL,
    prs_found   INTEGER DEFAULT 0,
    pods_found  INTEGER DEFAULT 0,
    status      TEXT,
    notes       TEXT
);
"""

@contextmanager
def get_conn():
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    try:
        yield conn
        conn.commit()
    finally:
        conn.close()

def init_db():
    with get_conn() as conn:
        conn.executescript(SCHEMA)

# ── Upsert helpers ────────────────────────────────────────────────────────────

def upsert_press_release(title: str, url: str, date: str = None,
                          category: str = None, description: str = None) -> int:
    """Insert or update a press release. Returns the row id."""
    now = datetime.utcnow().isoformat()
    with get_conn() as conn:
        cur = conn.execute(
            """
            INSERT INTO press_releases (title, url, date, category, description, scraped_at)
            VALUES (?, ?, ?, ?, ?, ?)
            ON CONFLICT(url) DO UPDATE SET
                title       = excluded.title,
                date        = excluded.date,
                category    = excluded.category,
                description = excluded.description,
                scraped_at  = excluded.scraped_at
            """,
            (title, url, date, category, description, now),
        )
        if cur.lastrowid:
            return cur.lastrowid
        row = conn.execute("SELECT id FROM press_releases WHERE url = ?", (url,)).fetchone()
        return row["id"]

def upsert_podcast(title: str, url: str, pr_id: int = None,
                   audio_url: str = None, date: str = None, duration: str = None,
                   language: str = "en") -> int:
    now = datetime.utcnow().isoformat()
    with get_conn() as conn:
        cur = conn.execute(
            """
            INSERT INTO podcasts (pr_id, title, url, audio_url, date, duration, language, scraped_at)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(url) DO UPDATE SET
                title     = excluded.title,
                audio_url = excluded.audio_url,
                date      = excluded.date,
                duration  = excluded.duration,
                language  = excluded.language,
                scraped_at= excluded.scraped_at
            """,
            (pr_id, title, url, audio_url, date, duration, language, now),
        )
        if cur.lastrowid:
            return cur.lastrowid
        row = conn.execute("SELECT id FROM podcasts WHERE url = ?", (url,)).fetchone()
        return row["id"]

def log_scrape(prs_found: int, pods_found: int, status: str, notes: str = ""):
    with get_conn() as conn:
        conn.execute(
            "INSERT INTO scrape_log (ran_at, prs_found, pods_found, status, notes) VALUES (?,?,?,?,?)",
            (datetime.utcnow().isoformat(), prs_found, pods_found, status, notes),
        )

# ── Query helpers ─────────────────────────────────────────────────────────────

def get_all_press_releases():
    with get_conn() as conn:
        return conn.execute(
            """
            SELECT
                pr.id,
                pr.title,
                pr.url,
                pr.date,
                pr.category,
                pr.description,
                pr.scraped_at,
                COUNT(p.id) AS podcast_count
            FROM press_releases pr
            LEFT JOIN podcasts p ON p.pr_id = pr.id
            GROUP BY pr.id
            ORDER BY pr.date DESC, pr.id DESC
            """
        ).fetchall()

def get_all_podcasts():
    with get_conn() as conn:
        return conn.execute(
            """
            SELECT p.*, pr.title AS pr_title
            FROM podcasts p
            LEFT JOIN press_releases pr ON pr.id = p.pr_id
            ORDER BY p.date DESC, p.id DESC
            """
        ).fetchall()

def get_counts_by_day():
    with get_conn() as conn:
        return conn.execute(
            "SELECT date, COUNT(*) as count FROM press_releases WHERE date IS NOT NULL GROUP BY date ORDER BY date"
        ).fetchall()

def get_counts_by_month():
    with get_conn() as conn:
        return conn.execute(
            """
            SELECT substr(date,1,7) AS month, COUNT(*) AS count
            FROM press_releases WHERE date IS NOT NULL
            GROUP BY month ORDER BY month
            """
        ).fetchall()

def get_scrape_log():
    with get_conn() as conn:
        return conn.execute(
            "SELECT * FROM scrape_log ORDER BY ran_at DESC LIMIT 50"
        ).fetchall()

def get_stats():
    with get_conn() as conn:
        total_prs    = conn.execute("SELECT COUNT(*) FROM press_releases").fetchone()[0]
        total_pods   = conn.execute("SELECT COUNT(*) FROM podcasts").fetchone()[0]
        last_scraped = conn.execute("SELECT MAX(scraped_at) FROM press_releases").fetchone()[0]
        return {"total_prs": total_prs, "total_podcasts": total_pods, "last_scraped": last_scraped}

+def get_all_podcasts(language: str = None):

+                SELECT p.*, pr.title AS pr_title

+                LEFT JOIN press_releases pr ON pr.id = p.pr_id

+                ORDER BY p.date DESC, p.id DESC


Now let me also update the dashboard to show Arabic/English podcast filtering and fix the language detection:


            # Language detection (Arabic title?)
            is_arabic = bool(re.search(r'[\u0600-\u06FF]', title))
            language  = "ar" if is_arabic else "en"

            # Language: Arabic if title contains Arabic Unicode block
            is_arabic = bool(re.search(r'[\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF]', title))
            language  = "ar" if is_arabic else "en"


The file /Users/khalidbinkalban/Projects/dewa-pr-tracker/scraper.py has been updated successfully.

            # Language: Arabic if title contains Arabic Unicode block
            is_arabic = bool(re.search(r'[\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF]', title))
            language  = "ar" if is_arabic else "en"

"""
DEWA PR Tracker — Scraper
  - Press Releases: Playwright (Chromium) — iterates all years via year filter
  - Podcasts:       Direct RSS feed parse from media.rss.com (no browser needed)

Usage:
  python scraper.py              # normal run
  python scraper.py --debug      # visible browser + saves debug HTML/screenshots
  python scraper.py --visible    # visible browser only
"""
import argparse
import re
import xml.etree.ElementTree as ET
from datetime import datetime
from email.utils import parsedate_to_datetime
from pathlib import Path
from urllib.request import urlopen, Request

from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout

import database as db
from config import (
    DEWA_NEWS_URL, DEWA_BASE_URL,
    SCRAPER_HEADLESS, SCRAPER_TIMEOUT_MS, SCRAPER_WAIT_AFTER_LOAD_MS,
)

RSS_FEED_URL = "https://media.rss.com/dewa-news/feed.xml"

# Years available on the DEWA news page
SCRAPE_YEARS = ["2019", "2020", "2021", "2022", "2023", "2024", "2025", "2026"]

# ── Utilities ─────────────────────────────────────────────────────────────────

def normalise_url(href: str) -> str:
    if not href:
        return ""
    href = href.strip()
    if href.startswith("http"):
        return href
    if href.startswith("/"):
        return DEWA_BASE_URL + href
    return href

def parse_dewa_date(raw: str) -> str:
    """Parse DEWA date format '30 March 2026' → '2026-03-30'."""
    if not raw:
        return None
    raw = raw.strip()
    try:
        return datetime.strptime(raw, "%d %B %Y").strftime("%Y-%m-%d")
    except ValueError:
        pass
    try:
        return datetime.strptime(raw, "%d %b %Y").strftime("%Y-%m-%d")
    except ValueError:
        pass
    m = re.search(r"(\d{4})", raw)
    return m.group(1) + "-01-01" if m else None

def seconds_to_mmss(secs) -> str:
    try:
        s = int(secs)
        return f"{s // 60}:{s % 60:02d}"
    except (TypeError, ValueError):
        return str(secs) if secs else None

# ── Press Release scraper (Playwright) ───────────────────────────────────────

def extract_items_from_page(page) -> list:
    """Extract all .m9-teaser--news_all items currently visible on the page."""
    items = page.query_selector_all(".m9-teaser--news_all")
    results = []
    for item in items:
        try:
            # Link — the whole card is an
            link_el = item.query_selector("a.teaser__link")
            if not link_el:
                link_el = item.query_selector("a")
            href  = link_el.get_attribute("href") if link_el else ""
            url   = normalise_url(href)

            # Title
            title_el = item.query_selector("h4.teaser__title")
            if not title_el:
                title_el = item.query_selector("h2, h3, h4, h5, .title")
            title = title_el.inner_text().strip() if title_el else ""

            if not title or not url:
                continue

            # Date — "30 March 2026"
            date_el = item.query_selector("h6.teaser__date")
            raw_date = date_el.inner_text().strip() if date_el else ""
            date = parse_dewa_date(raw_date)

            # Description — stored in data-vl attribute of the

            desc_el = item.query_selector("p.teaser__copy")
            description = None
            if desc_el:
                description = (
                    desc_el.get_attribute("data-vl")
                    or desc_el.inner_text().strip()
                    or None
                )

            results.append({
                "title": title,
                "url": url,
                "date": date,
                "description": description,
                "category": None,
            })
        except Exception as e:
            print(f"  [warn] Skipping item: {e}")
    return results

def scrape_press_releases(page, debug: bool = False) -> list:
    print(f"[scraper] → News page: {DEWA_NEWS_URL}")
    page.goto(DEWA_NEWS_URL, timeout=SCRAPER_TIMEOUT_MS, wait_until="networkidle")
    page.wait_for_timeout(SCRAPER_WAIT_AFTER_LOAD_MS)

    if debug:
        page.screenshot(path="debug_news.png", full_page=True)
        Path("debug_news.html").write_text(page.content(), encoding="utf-8")
        print("[debug] Saved debug_news.png + debug_news.html")

    all_results = []
    seen_urls   = set()

    month_select = page.query_selector("select[name='month']")
    year_select  = page.query_selector("select[name='year']")

    if not year_select:
        print("[scraper] WARNING: year selector not found — extracting current page only")
        return extract_items_from_page(page)

    for year in SCRAPE_YEARS:
        print(f"  [scraper] Year {year}:", end=" ", flush=True)
        try:
            # IMPORTANT: reset month first (fires API call), then select year
            if month_select:
                month_select.select_option(value="")
                page.wait_for_timeout(2000)
            year_select.select_option(value=year)
            page.wait_for_timeout(2500)

            # Click "More" until it disappears — loads all pages for this year
            pages_loaded = 1
            while True:
                more_btn = page.query_selector("#loadmore")
                if more_btn and more_btn.is_visible():
                    try:
                        more_btn.scroll_into_view_if_needed()
                        more_btn.click()
                        page.wait_for_timeout(2500)
                        pages_loaded += 1
                    except Exception:
                        break
                else:
                    break

            items     = extract_items_from_page(page)
            new_items = [i for i in items if i["url"] not in seen_urls]
            for i in new_items:
                seen_urls.add(i["url"])
            all_results.extend(new_items)
            print(f"{len(new_items)} items ({pages_loaded} page(s) loaded)")

        except Exception as e:
            print(f"error: {e}")
            continue

    print(f"[scraper] Total press releases extracted: {len(all_results)}")
    return all_results

# ── Podcast scraper (RSS feed) ────────────────────────────────────────────────

def scrape_podcasts_from_rss() -> list:
    print(f"[scraper] → Fetching RSS feed: {RSS_FEED_URL}")
    try:
        req = Request(
            RSS_FEED_URL,
            headers={"User-Agent": "Mozilla/5.0 (compatible; DEWATracker/1.0)"}
        )
        with urlopen(req, timeout=30) as resp:
            raw_xml = resp.read()
    except Exception as e:
        print(f"[scraper] RSS fetch error: {e}")
        return []

    # Parse XML
    ns = {
        "itunes":  "http://www.itunes.com/dtds/podcast-1.0.dtd",
        "podcast": "https://podcastindex.org/namespace/1.0",
        "content": "http://purl.org/rss/1.0/modules/content/",
    }

    try:
        root = ET.fromstring(raw_xml)
    except ET.ParseError as e:
        print(f"[scraper] XML parse error: {e}")
        return []

    channel = root.find("channel")
    if not channel:
        return []

    results = []
    for item in channel.findall("item"):
        try:
            title = (item.findtext("title") or "").strip()
            link  = (item.findtext("link")  or "").strip()

            if not title or not link:
                continue

            # Date — RFC 822 format e.g. "Wed, 01 Apr 2026 08:58:31 GMT"
            pub_date_raw = item.findtext("pubDate") or ""
            date = None
            if pub_date_raw:
                try:
                    date = parsedate_to_datetime(pub_date_raw).strftime("%Y-%m-%d")
                except Exception:
                    pass

            # MP3 URL from <enclosure>
            enclosure = item.find("enclosure")
            audio_url = enclosure.get("url") if enclosure is not None else None

            # Duration (seconds → mm:ss)
            dur_raw  = item.findtext("itunes:duration", namespaces=ns) or ""
            duration = seconds_to_mmss(dur_raw) if dur_raw else None

            # Description
            desc_raw = item.findtext("description") or ""
            desc_raw = re.sub(r"<[^>]+>", "", desc_raw).strip()
            description = desc_raw or None

            # Language detection (Arabic title?)
            is_arabic = bool(re.search(r'[\u0600-\u06FF]', title))
            language  = "ar" if is_arabic else "en"

            results.append({
                "title":       title,
                "url":         link,
                "audio_url":   audio_url,
                "date":        date,
                "duration":    duration,
                "description": description,
                "language":    language,
            })
        except Exception as e:
            print(f"  [warn] Skipping podcast item: {e}")
            continue

    print(f"[scraper] Total podcast episodes from RSS: {len(results)}")
    return results

# ── Match podcasts → press releases by title ──────────────────────────────────

def normalise_title(t: str) -> str:
    """Lowercase, strip punctuation for fuzzy matching."""
    t = t.lower()
    t = re.sub(r"[^\w\s]", " ", t)
    t = re.sub(r"\s+", " ", t).strip()
    return t

def match_podcasts_to_prs(podcasts: list, pr_id_map: dict) -> list:
    """
    pr_id_map: { url: (id, normalised_title) }
    Match by:
    1. URL slug — rss.com episode URL may contain matching text
    2. Title substring overlap (>60% word overlap)
    """
    pr_titles = [(pid, ntitle) for _, (pid, ntitle) in pr_id_map.items()]

    matched = []
    for pod in podcasts:
        pr_id = None
        pod_norm = normalise_title(pod["title"])
        pod_words = set(pod_norm.split())

        best_score = 0
        best_pid   = None
        for pid, pr_norm in pr_titles:
            pr_words = set(pr_norm.split())
            if not pr_words:
                continue
            overlap = len(pod_words & pr_words) / len(pr_words)
            if overlap > best_score:
                best_score = overlap
                best_pid   = pid

        if best_score >= 0.5:
            pr_id = best_pid

        matched.append({**pod, "pr_id": pr_id})

    linked = sum(1 for p in matched if p["pr_id"])
    print(f"[scraper] Podcasts linked to PRs: {linked}/{len(matched)}")
    return matched

# ── Main ──────────────────────────────────────────────────────────────────────

def run_scrape(debug: bool = False, headless: bool = None) -> tuple[int, int]:
    if headless is None:
        headless = SCRAPER_HEADLESS
    if debug:
        headless = False

    db.init_db()

    prs_saved  = 0
    pods_saved = 0
    status     = "success"
    notes      = ""

    try:
        # ── 1. Press Releases ─────────────────────────────────────────────────
        with sync_playwright() as pw:
            browser = pw.chromium.launch(
                headless=headless,
                args=["--disable-blink-features=AutomationControlled"],
            )
            ctx = browser.new_context(
                user_agent=(
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
                    "AppleWebKit/537.36 (KHTML, like Gecko) "
                    "Chrome/122.0.0.0 Safari/537.36"
                ),
                viewport={"width": 1440, "height": 900},
            )
            page = ctx.new_page()
            pr_data = scrape_press_releases(page, debug=debug)
            browser.close()

        pr_id_map = {}
        for pr in pr_data:
            pid = db.upsert_press_release(
                title=pr["title"],
                url=pr["url"],
                date=pr["date"],
                category=pr.get("category"),
                description=pr.get("description"),
            )
            pr_id_map[pr["url"]] = (pid, normalise_title(pr["title"]))
            prs_saved += 1

        # ── 2. Podcasts from RSS feed ─────────────────────────────────────────
        pod_data = scrape_podcasts_from_rss()
        pod_data = match_podcasts_to_prs(pod_data, pr_id_map)
        for pod in pod_data:
            db.upsert_podcast(
                title=pod["title"],
                url=pod["url"],
                pr_id=pod.get("pr_id"),
                audio_url=pod.get("audio_url"),
                date=pod.get("date"),
                duration=pod.get("duration"),
            )
            pods_saved += 1

    except Exception as e:
        status = "error"
        notes  = str(e)
        print(f"[scraper] ERROR: {e}")

    db.log_scrape(prs_saved, pods_saved, status, notes)
    print(f"\n✅ Done — {prs_saved} PRs, {pods_saved} podcasts saved. Status: {status}")
    if notes:
        print(f"   Notes: {notes}")

    return prs_saved, pods_saved

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="DEWA PR & Podcast Scraper")
    parser.add_argument("--debug",   action="store_true", help="Visible browser + save debug files")
    parser.add_argument("--visible", action="store_true", help="Visible browser only")
    args = parser.parse_args()

    run_scrape(debug=args.debug, headless=not args.visible and SCRAPER_HEADLESS)

-            # Language detection (Arabic title?)

-            is_arabic = bool(re.search(r'[\u0600-\u06FF]', title))

+            # Language: Arabic if title contains Arabic Unicode block

+            is_arabic = bool(re.search(r'[\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF]', title))


Now let's update the dashboard to add language filtering for podcasts and start it:


prs, pods, stats, scrape_log = load_data()
df_pr  = pd.DataFrame(prs)
df_pod = pd.DataFrame(pods)

prs, pods, stats, scrape_log = load_data()
df_pr  = pd.DataFrame(prs)
df_pod = pd.DataFrame(pods)

# Fix language field in podcasts (re-detect from title)
if not df_pod.empty and "title" in df_pod.columns:
    import re as _re
    df_pod["language"] = df_pod["title"].apply(
        lambda t: "ar" if _re.search(r'[\u0600-\u06FF]', str(t)) else "en"
    )


The file /Users/khalidbinkalban/Projects/dewa-pr-tracker/dashboard.py has been updated successfully.

prs, pods, stats, scrape_log = load_data()
df_pr  = pd.DataFrame(prs)
df_pod = pd.DataFrame(pods)

# Fix language field in podcasts (re-detect from title)
if not df_pod.empty and "title" in df_pod.columns:
    import re as _re
    df_pod["language"] = df_pod["title"].apply(
        lambda t: "ar" if _re.search(r'[\u0600-\u06FF]', str(t)) else "en"
    )

"""
DEWA PR Tracker — Streamlit Dashboard
Run:  streamlit run dashboard.py
"""
import math
from datetime import datetime, timedelta

import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
import streamlit as st

import database as db
from config import (
    COST_PER_PODCAST_OLD_AED,
    NOTEBOOKLM_MONTHLY_USD,
    USD_TO_AED,
    DASHBOARD_TITLE,
    PAGE_SIZE,
)

# ── Page setup ────────────────────────────────────────────────────────────────
st.set_page_config(
    page_title=DASHBOARD_TITLE,
    page_icon="📰",
    layout="wide",
    initial_sidebar_state="expanded",
)

# ── Custom CSS ────────────────────────────────────────────────────────────────
st.markdown("""

""", unsafe_allow_html=True)

# ── Data loading (cached 5 min) ───────────────────────────────────────────────
@st.cache_data(ttl=300)
def load_data():
    db.init_db()
    prs   = [dict(r) for r in db.get_all_press_releases()]
    pods  = [dict(r) for r in db.get_all_podcasts()]
    stats = db.get_stats()
    log   = [dict(r) for r in db.get_scrape_log()]
    return prs, pods, stats, log

prs, pods, stats, scrape_log = load_data()
df_pr  = pd.DataFrame(prs)
df_pod = pd.DataFrame(pods)

if not df_pr.empty:
    df_pr["date"] = pd.to_datetime(df_pr["date"], errors="coerce")
    df_pr["week"]  = df_pr["date"].dt.to_period("W").astype(str)
    df_pr["month"] = df_pr["date"].dt.to_period("M").astype(str)
    df_pr["year"]  = df_pr["date"].dt.year

if not df_pod.empty:
    df_pod["date"] = pd.to_datetime(df_pod["date"], errors="coerce")
    df_pod["month"] = df_pod["date"].dt.to_period("M").astype(str)

# ── Sidebar ───────────────────────────────────────────────────────────────────
with st.sidebar:
    st.image("https://www.dewa.gov.ae/~/media/Images/Header/DEWA-Logo.ashx", width=160)
    st.title("Filters")

    # Date range
    if not df_pr.empty and df_pr["date"].notna().any():
        min_date = df_pr["date"].min().date()
        max_date = df_pr["date"].max().date()
    else:
        min_date = datetime(2020, 1, 1).date()
        max_date = datetime.today().date()

    date_from, date_to = st.date_input(
        "Date range",
        value=(min_date, max_date),
        min_value=min_date,
        max_value=max_date,
    )

    # Category filter
    categories = ["All"]
    if not df_pr.empty and "category" in df_pr.columns:
        cats = df_pr["category"].dropna().unique().tolist()
        categories += sorted(cats)
    selected_cat = st.selectbox("Category", categories)

    # Podcast filter
    has_podcast_filter = st.checkbox("Show only PRs with podcast", value=False)

    st.divider()
    if st.button("🔄 Refresh data", use_container_width=True):
        st.cache_data.clear()
        st.rerun()

    if st.button("▶ Run scraper now", use_container_width=True):
        with st.spinner("Scraping DEWA website…"):
            try:
                from scraper import run_scrape
                prs_n, pods_n = run_scrape()
                st.success(f"Done! {prs_n} PRs, {pods_n} podcasts saved.")
                st.cache_data.clear()
                st.rerun()
            except Exception as e:
                st.error(f"Scraper error: {e}")

    st.divider()
    st.caption(f"Last scraped: {stats.get('last_scraped', 'Never') or 'Never'}")

# ── Apply filters ─────────────────────────────────────────────────────────────
filtered = df_pr.copy()
if not filtered.empty:
    if filtered["date"].notna().any():
        filtered = filtered[
            (filtered["date"].dt.date >= date_from) &
            (filtered["date"].dt.date <= date_to)
        ]
    if selected_cat != "All":
        filtered = filtered[filtered["category"] == selected_cat]
    if has_podcast_filter:
        filtered = filtered[filtered["podcast_count"] > 0]

# ═══════════════════════════════════════════════════════════════════════════════
# HEADER
# ═══════════════════════════════════════════════════════════════════════════════
st.title(f"📰 {DASHBOARD_TITLE}")
st.markdown(f"*Tracking DEWA press releases and podcast ROI — {datetime.today().strftime('%d %B %Y')}*")
st.divider()

# ═══════════════════════════════════════════════════════════════════════════════
# ROW 1 — Key Metrics
# ═══════════════════════════════════════════════════════════════════════════════
col1, col2, col3, col4, col5 = st.columns(5)

total_prs  = len(filtered)
total_pods = stats.get("total_podcasts", 0)

today = datetime.today().date()
week_start  = today - timedelta(days=today.weekday())
month_start = today.replace(day=1)

if not filtered.empty and filtered["date"].notna().any():
    prs_today   = len(filtered[filtered["date"].dt.date == today])
    prs_week    = len(filtered[filtered["date"].dt.date >= week_start])
    prs_month   = len(filtered[filtered["date"].dt.date >= month_start])
else:
    prs_today = prs_week = prs_month = 0

with col1:
    st.metric("Total Press Releases", f"{total_prs:,}")
with col2:
    st.metric("Total Podcasts", f"{total_pods:,}")
with col3:
    st.metric("Published Today", prs_today)
with col4:
    st.metric("This Week", prs_week)
with col5:
    st.metric("This Month", prs_month)

st.divider()

# ═══════════════════════════════════════════════════════════════════════════════
# ROW 2 — ROI Calculator
# ═══════════════════════════════════════════════════════════════════════════════
st.subheader("💰 Podcast ROI Calculator")

notebooklm_aed = NOTEBOOKLM_MONTHLY_USD * USD_TO_AED

# Calculate months since NotebookLM adoption (user can adjust in the UI)
c_left, c_right = st.columns([2, 1])
with c_right:
    months_using = st.number_input(
        "Months using NotebookLM", min_value=1, max_value=60, value=6
    )
    podcasts_in_period = st.number_input(
        "Podcasts produced in that period", min_value=0, max_value=500,
        value=total_pods
    )

old_cost  = podcasts_in_period * COST_PER_PODCAST_OLD_AED
new_cost  = months_using * notebooklm_aed
savings   = old_cost - new_cost
roi_pct   = (savings / old_cost * 100) if old_cost > 0 else 0

with c_left:
    r1, r2, r3 = st.columns(3)
    with r1:
        st.markdown("**Before — Traditional Production**")
        st.markdown(f"AED {old_cost:,.0f}", unsafe_allow_html=True)
        st.caption(f"{podcasts_in_period} podcast(s) × AED {COST_PER_PODCAST_OLD_AED:,} each")

    with r2:
        st.markdown("**After — NotebookLM**")
        st.markdown(f"AED {new_cost:,.0f}", unsafe_allow_html=True)
        st.caption(f"{months_using} month(s) × AED {notebooklm_aed:.0f}/mo (${NOTEBOOKLM_MONTHLY_USD}/mo)")

    with r3:
        st.markdown("**Net Savings**")
        st.markdown(f"AED {savings:,.0f}", unsafe_allow_html=True)
        st.caption(f"ROI: {roi_pct:.1f}%")

    # ROI gauge
    fig_gauge = go.Figure(go.Indicator(
        mode="gauge+number+delta",
        value=roi_pct,
        title={"text": "ROI %"},
        delta={"reference": 0},
        gauge={
            "axis": {"range": [0, 100]},
            "bar": {"color": "#00c853"},
            "steps": [
                {"range": [0, 50], "color": "#1e2a3a"},
                {"range": [50, 80], "color": "#0d3b2e"},
                {"range": [80, 100], "color": "#0a4a35"},
            ],
            "threshold": {"line": {"color": "gold", "width": 4}, "value": 90},
        },
        number={"suffix": "%"},
    ))
    fig_gauge.update_layout(height=220, margin=dict(t=30, b=0, l=20, r=20),
                            paper_bgcolor="rgba(0,0,0,0)", font_color="white")
    st.plotly_chart(fig_gauge, use_container_width=True)

st.divider()

# ═══════════════════════════════════════════════════════════════════════════════
# ROW 3 — Publication Charts
# ═══════════════════════════════════════════════════════════════════════════════
st.subheader("📊 Publication Trends")
tab_daily, tab_weekly, tab_monthly = st.tabs(["Daily", "Weekly", "Monthly"])

def make_bar(df_grouped, x, y, title, color="#00b4d8"):
    fig = px.bar(df_grouped, x=x, y=y, title=title,
                 color_discrete_sequence=[color])
    fig.update_layout(
        paper_bgcolor="rgba(0,0,0,0)",
        plot_bgcolor="rgba(0,0,0,0)",
        font_color="white",
        xaxis=dict(showgrid=False),
        yaxis=dict(gridcolor="#2a3a4a"),
        margin=dict(t=40, b=10),
    )
    return fig

with tab_daily:
    if not filtered.empty and filtered["date"].notna().any():
        daily = filtered.groupby(filtered["date"].dt.date).size().reset_index(name="count")
        daily.columns = ["date", "count"]
        st.plotly_chart(make_bar(daily, "date", "count", "Press Releases per Day"), use_container_width=True)
    else:
        st.info("No date data available yet. Run the scraper first.")

with tab_weekly:
    if not filtered.empty and "week" in filtered.columns and filtered["date"].notna().any():
        weekly = filtered.groupby("week").size().reset_index(name="count")
        st.plotly_chart(make_bar(weekly, "week", "count", "Press Releases per Week", "#4cc9f0"), use_container_width=True)
    else:
        st.info("No data yet.")

with tab_monthly:
    if not filtered.empty and "month" in filtered.columns and filtered["date"].notna().any():
        monthly = filtered.groupby("month").size().reset_index(name="count")
        st.plotly_chart(make_bar(monthly, "month", "count", "Press Releases per Month", "#7209b7"), use_container_width=True)
    else:
        st.info("No data yet.")

st.divider()

# ═══════════════════════════════════════════════════════════════════════════════
# ROW 4 — Press Releases Table
# ═══════════════════════════════════════════════════════════════════════════════
st.subheader("📋 Press Releases")

search = st.text_input("Search press releases", placeholder="Type to filter by title…")
if search:
    display = filtered[filtered["title"].str.contains(search, case=False, na=False)]
else:
    display = filtered.copy()

total_pages = max(1, math.ceil(len(display) / PAGE_SIZE))
page_num = st.number_input("Page", min_value=1, max_value=total_pages, value=1, step=1)
start = (page_num - 1) * PAGE_SIZE
end   = start + PAGE_SIZE
page_data = display.iloc[start:end]

st.caption(f"Showing {start+1}–{min(end, len(display))} of {len(display)} press releases (page {page_num}/{total_pages})")

for _, row in page_data.iterrows():
    with st.container():
        tc, dc, pc = st.columns([5, 2, 1])
        with tc:
            url   = row.get("url", "")
            title = row.get("title", "—")
            st.markdown(f"**[{title}]({url})**" if url else f"**{title}**")
            if row.get("description"):
                st.caption(row["description"][:160] + "…" if len(str(row["description"])) > 160 else row["description"])
        with dc:
            d = row.get("date")
            st.markdown(f"`{d.strftime('%d %b %Y') if pd.notna(d) else '—'}`")
            if row.get("category"):
                st.caption(row["category"])
        with pc:
            pc_count = row.get("podcast_count", 0)
            if pc_count:
                st.markdown(f"🎙 {pc_count}")
            else:
                st.markdown("—")
    st.divider()

# ═══════════════════════════════════════════════════════════════════════════════
# ROW 5 — Podcasts Table
# ═══════════════════════════════════════════════════════════════════════════════
if not df_pod.empty:
    st.subheader("🎙 Podcasts")
    for _, pod in df_pod.iterrows():
        with st.expander(f"{pod.get('title', '—')}  |  {pod.get('date', '')}"):
            col_a, col_b = st.columns([3, 1])
            with col_a:
                if pod.get("pr_title"):
                    st.markdown(f"**Linked PR:** {pod['pr_title']}")
                if pod.get("url"):
                    st.markdown(f"[Open podcast page]({pod['url']})")
                if pod.get("audio_url"):
                    st.audio(pod["audio_url"])
            with col_b:
                if pod.get("duration"):
                    st.metric("Duration", pod["duration"])
                cost_saved = COST_PER_PODCAST_OLD_AED
                st.metric("Saved vs old cost", f"AED {cost_saved:,}")

st.divider()

# ═══════════════════════════════════════════════════════════════════════════════
# ROW 6 — Scrape Log
# ═══════════════════════════════════════════════════════════════════════════════
with st.expander("🔍 Scrape history"):
    if scrape_log:
        df_log = pd.DataFrame(scrape_log)
        st.dataframe(df_log[["ran_at", "prs_found", "pods_found", "status", "notes"]],
                     use_container_width=True)
    else:
        st.info("No scrape runs recorded yet.")

st.caption("DEWA PR Tracker · Built with Playwright + SQLite + Streamlit")

+# Fix language field in podcasts (re-detect from title)

+if not df_pod.empty and "title" in df_pod.columns:

+    df_pod["language"] = df_pod["title"].apply(

+        lambda t: "ar" if _re.search(r'[\u0600-\u06FF]', str(t)) else "en"


if not df_pod.empty:
    st.subheader("🎙 Podcasts")
    for _, pod in df_pod.iterrows():

if not df_pod.empty:
    st.subheader("🎙 Podcasts")

    pod_lang_filter = st.radio(
        "Language", ["All", "English (EN)", "Arabic (AR)"],
        horizontal=True,
    )
    df_pod_show = df_pod.copy()
    if pod_lang_filter == "English (EN)":
        df_pod_show = df_pod_show[df_pod_show["language"] == "en"]
    elif pod_lang_filter == "Arabic (AR)":
        df_pod_show = df_pod_show[df_pod_show["language"] == "ar"]

    pod_search = st.text_input("Search podcasts", placeholder="Filter by title…")
    if pod_search:
        df_pod_show = df_pod_show[df_pod_show["title"].str.contains(pod_search, case=False, na=False)]

    st.caption(f"Showing {len(df_pod_show)} of {len(df_pod)} episodes")

    for _, pod in df_pod_show.head(100).iterrows():


if not df_pod.empty:
    st.subheader("🎙 Podcasts")

    pod_lang_filter = st.radio(
        "Language", ["All", "English (EN)", "Arabic (AR)"],
        horizontal=True,
    )
    df_pod_show = df_pod.copy()
    if pod_lang_filter == "English (EN)":
        df_pod_show = df_pod_show[df_pod_show["language"] == "en"]
    elif pod_lang_filter == "Arabic (AR)":
        df_pod_show = df_pod_show[df_pod_show["language"] == "ar"]

    pod_search = st.text_input("Search podcasts", placeholder="Filter by title…")
    if pod_search:
        df_pod_show = df_pod_show[df_pod_show["title"].str.contains(pod_search, case=False, na=False)]

    st.caption(f"Showing {len(df_pod_show)} of {len(df_pod)} episodes")

    for _, pod in df_pod_show.head(100).iterrows():

"""
DEWA PR Tracker — Streamlit Dashboard
Run:  streamlit run dashboard.py
"""
import math
from datetime import datetime, timedelta

import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
import streamlit as st

import database as db
from config import (
    COST_PER_PODCAST_OLD_AED,
    NOTEBOOKLM_MONTHLY_USD,
    USD_TO_AED,
    DASHBOARD_TITLE,
    PAGE_SIZE,
)

# ── Page setup ────────────────────────────────────────────────────────────────
st.set_page_config(
    page_title=DASHBOARD_TITLE,
    page_icon="📰",
    layout="wide",
    initial_sidebar_state="expanded",
)

# ── Custom CSS ────────────────────────────────────────────────────────────────
st.markdown("""

""", unsafe_allow_html=True)

# ── Data loading (cached 5 min) ───────────────────────────────────────────────
@st.cache_data(ttl=300)
def load_data():
    db.init_db()
    prs   = [dict(r) for r in db.get_all_press_releases()]
    pods  = [dict(r) for r in db.get_all_podcasts()]
    stats = db.get_stats()
    log   = [dict(r) for r in db.get_scrape_log()]
    return prs, pods, stats, log

prs, pods, stats, scrape_log = load_data()
df_pr  = pd.DataFrame(prs)
df_pod = pd.DataFrame(pods)

# Fix language field in podcasts (re-detect from title)
if not df_pod.empty and "title" in df_pod.columns:
    import re as _re
    df_pod["language"] = df_pod["title"].apply(
        lambda t: "ar" if _re.search(r'[\u0600-\u06FF]', str(t)) else "en"
    )

if not df_pr.empty:
    df_pr["date"] = pd.to_datetime(df_pr["date"], errors="coerce")
    df_pr["week"]  = df_pr["date"].dt.to_period("W").astype(str)
    df_pr["month"] = df_pr["date"].dt.to_period("M").astype(str)
    df_pr["year"]  = df_pr["date"].dt.year

if not df_pod.empty:
    df_pod["date"] = pd.to_datetime(df_pod["date"], errors="coerce")
    df_pod["month"] = df_pod["date"].dt.to_period("M").astype(str)

# ── Sidebar ───────────────────────────────────────────────────────────────────
with st.sidebar:
    st.image("https://www.dewa.gov.ae/~/media/Images/Header/DEWA-Logo.ashx", width=160)
    st.title("Filters")

    # Date range
    if not df_pr.empty and df_pr["date"].notna().any():
        min_date = df_pr["date"].min().date()
        max_date = df_pr["date"].max().date()
    else:
        min_date = datetime(2020, 1, 1).date()
        max_date = datetime.today().date()

    date_from, date_to = st.date_input(
        "Date range",
        value=(min_date, max_date),
        min_value=min_date,
        max_value=max_date,
    )

    # Category filter
    categories = ["All"]
    if not df_pr.empty and "category" in df_pr.columns:
        cats = df_pr["category"].dropna().unique().tolist()
        categories += sorted(cats)
    selected_cat = st.selectbox("Category", categories)

    # Podcast filter
    has_podcast_filter = st.checkbox("Show only PRs with podcast", value=False)

    st.divider()
    if st.button("🔄 Refresh data", use_container_width=True):
        st.cache_data.clear()
        st.rerun()

    if st.button("▶ Run scraper now", use_container_width=True):
        with st.spinner("Scraping DEWA website…"):
            try:
                from scraper import run_scrape
                prs_n, pods_n = run_scrape()
                st.success(f"Done! {prs_n} PRs, {pods_n} podcasts saved.")
                st.cache_data.clear()
                st.rerun()
            except Exception as e:
                st.error(f"Scraper error: {e}")

    st.divider()
    st.caption(f"Last scraped: {stats.get('last_scraped', 'Never') or 'Never'}")

# ── Apply filters ─────────────────────────────────────────────────────────────
filtered = df_pr.copy()
if not filtered.empty:
    if filtered["date"].notna().any():
        filtered = filtered[
            (filtered["date"].dt.date >= date_from) &
            (filtered["date"].dt.date <= date_to)
        ]
    if selected_cat != "All":
        filtered = filtered[filtered["category"] == selected_cat]
    if has_podcast_filter:
        filtered = filtered[filtered["podcast_count"] > 0]

# ═══════════════════════════════════════════════════════════════════════════════
# HEADER
# ═══════════════════════════════════════════════════════════════════════════════
st.title(f"📰 {DASHBOARD_TITLE}")
st.markdown(f"*Tracking DEWA press releases and podcast ROI — {datetime.today().strftime('%d %B %Y')}*")
st.divider()

# ═══════════════════════════════════════════════════════════════════════════════
# ROW 1 — Key Metrics
# ═══════════════════════════════════════════════════════════════════════════════
col1, col2, col3, col4, col5 = st.columns(5)

total_prs  = len(filtered)
total_pods = stats.get("total_podcasts", 0)

today = datetime.today().date()
week_start  = today - timedelta(days=today.weekday())
month_start = today.replace(day=1)

if not filtered.empty and filtered["date"].notna().any():
    prs_today   = len(filtered[filtered["date"].dt.date == today])
    prs_week    = len(filtered[filtered["date"].dt.date >= week_start])
    prs_month   = len(filtered[filtered["date"].dt.date >= month_start])
else:
    prs_today = prs_week = prs_month = 0

with col1:
    st.metric("Total Press Releases", f"{total_prs:,}")
with col2:
    st.metric("Total Podcasts", f"{total_pods:,}")
with col3:
    st.metric("Published Today", prs_today)
with col4:
    st.metric("This Week", prs_week)
with col5:
    st.metric("This Month", prs_month)

st.divider()

# ═══════════════════════════════════════════════════════════════════════════════
# ROW 2 — ROI Calculator
# ═══════════════════════════════════════════════════════════════════════════════
st.subheader("💰 Podcast ROI Calculator")

notebooklm_aed = NOTEBOOKLM_MONTHLY_USD * USD_TO_AED

# Calculate months since NotebookLM adoption (user can adjust in the UI)
c_left, c_right = st.columns([2, 1])
with c_right:
    months_using = st.number_input(
        "Months using NotebookLM", min_value=1, max_value=60, value=6
    )
    podcasts_in_period = st.number_input(
        "Podcasts produced in that period", min_value=0, max_value=500,
        value=total_pods
    )

old_cost  = podcasts_in_period * COST_PER_PODCAST_OLD_AED
new_cost  = months_using * notebooklm_aed
savings   = old_cost - new_cost
roi_pct   = (savings / old_cost * 100) if old_cost > 0 else 0

with c_left:
    r1, r2, r3 = st.columns(3)
    with r1:
        st.markdown("**Before — Traditional Production**")
        st.markdown(f"AED {old_cost:,.0f}", unsafe_allow_html=True)
        st.caption(f"{podcasts_in_period} podcast(s) × AED {COST_PER_PODCAST_OLD_AED:,} each")

    with r2:
        st.markdown("**After — NotebookLM**")
        st.markdown(f"AED {new_cost:,.0f}", unsafe_allow_html=True)
        st.caption(f"{months_using} month(s) × AED {notebooklm_aed:.0f}/mo (${NOTEBOOKLM_MONTHLY_USD}/mo)")

    with r3:
        st.markdown("**Net Savings**")
        st.markdown(f"AED {savings:,.0f}", unsafe_allow_html=True)
        st.caption(f"ROI: {roi_pct:.1f}%")

    # ROI gauge
    fig_gauge = go.Figure(go.Indicator(
        mode="gauge+number+delta",
        value=roi_pct,
        title={"text": "ROI %"},
        delta={"reference": 0},
        gauge={
            "axis": {"range": [0, 100]},
            "bar": {"color": "#00c853"},
            "steps": [
                {"range": [0, 50], "color": "#1e2a3a"},
                {"range": [50, 80], "color": "#0d3b2e"},
                {"range": [80, 100], "color": "#0a4a35"},
            ],
            "threshold": {"line": {"color": "gold", "width": 4}, "value": 90},
        },
        number={"suffix": "%"},
    ))
    fig_gauge.update_layout(height=220, margin=dict(t=30, b=0, l=20, r=20),
                            paper_bgcolor="rgba(0,0,0,0)", font_color="white")
    st.plotly_chart(fig_gauge, use_container_width=True)

st.divider()

# ═══════════════════════════════════════════════════════════════════════════════
# ROW 3 — Publication Charts
# ═══════════════════════════════════════════════════════════════════════════════
st.subheader("📊 Publication Trends")
tab_daily, tab_weekly, tab_monthly = st.tabs(["Daily", "Weekly", "Monthly"])

def make_bar(df_grouped, x, y, title, color="#00b4d8"):
    fig = px.bar(df_grouped, x=x, y=y, title=title,
                 color_discrete_sequence=[color])
    fig.update_layout(
        paper_bgcolor="rgba(0,0,0,0)",
        plot_bgcolor="rgba(0,0,0,0)",
        font_color="white",
        xaxis=dict(showgrid=False),
        yaxis=dict(gridcolor="#2a3a4a"),
        margin=dict(t=40, b=10),
    )
    return fig

with tab_daily:
    if not filtered.empty and filtered["date"].notna().any():
        daily = filtered.groupby(filtered["date"].dt.date).size().reset_index(name="count")
        daily.columns = ["date", "count"]
        st.plotly_chart(make_bar(daily, "date", "count", "Press Releases per Day"), use_container_width=True)
    else:
        st.info("No date data available yet. Run the scraper first.")

with tab_weekly:
    if not filtered.empty and "week" in filtered.columns and filtered["date"].notna().any():
        weekly = filtered.groupby("week").size().reset_index(name="count")
        st.plotly_chart(make_bar(weekly, "week", "count", "Press Releases per Week", "#4cc9f0"), use_container_width=True)
    else:
        st.info("No data yet.")

with tab_monthly:
    if not filtered.empty and "month" in filtered.columns and filtered["date"].notna().any():
        monthly = filtered.groupby("month").size().reset_index(name="count")
        st.plotly_chart(make_bar(monthly, "month", "count", "Press Releases per Month", "#7209b7"), use_container_width=True)
    else:
        st.info("No data yet.")

st.divider()

# ═══════════════════════════════════════════════════════════════════════════════
# ROW 4 — Press Releases Table
# ═══════════════════════════════════════════════════════════════════════════════
st.subheader("📋 Press Releases")

search = st.text_input("Search press releases", placeholder="Type to filter by title…")
if search:
    display = filtered[filtered["title"].str.contains(search, case=False, na=False)]
else:
    display = filtered.copy()

total_pages = max(1, math.ceil(len(display) / PAGE_SIZE))
page_num = st.number_input("Page", min_value=1, max_value=total_pages, value=1, step=1)
start = (page_num - 1) * PAGE_SIZE
end   = start + PAGE_SIZE
page_data = display.iloc[start:end]

st.caption(f"Showing {start+1}–{min(end, len(display))} of {len(display)} press releases (page {page_num}/{total_pages})")

for _, row in page_data.iterrows():
    with st.container():
        tc, dc, pc = st.columns([5, 2, 1])
        with tc:
            url   = row.get("url", "")
            title = row.get("title", "—")
            st.markdown(f"**[{title}]({url})**" if url else f"**{title}**")
            if row.get("description"):
                st.caption(row["description"][:160] + "…" if len(str(row["description"])) > 160 else row["description"])
        with dc:
            d = row.get("date")
            st.markdown(f"`{d.strftime('%d %b %Y') if pd.notna(d) else '—'}`")
            if row.get("category"):
                st.caption(row["category"])
        with pc:
            pc_count = row.get("podcast_count", 0)
            if pc_count:
                st.markdown(f"🎙 {pc_count}")
            else:
                st.markdown("—")
    st.divider()

# ═══════════════════════════════════════════════════════════════════════════════
# ROW 5 — Podcasts Table
# ═══════════════════════════════════════════════════════════════════════════════
if not df_pod.empty:
    st.subheader("🎙 Podcasts")
    for _, pod in df_pod.iterrows():
        with st.expander(f"{pod.get('title', '—')}  |  {pod.get('date', '')}"):
            col_a, col_b = st.columns([3, 1])
            with col_a:
                if pod.get("pr_title"):
                    st.markdown(f"**Linked PR:** {pod['pr_title']}")
                if pod.get("url"):
                    st.markdown(f"[Open podcast page]({pod['url']})")
                if pod.get("audio_url"):
                    st.audio(pod["audio_url"])
            with col_b:
                if pod.get("duration"):
                    st.metric("Duration", pod["duration"])
                cost_saved = COST_PER_PODCAST_OLD_AED
                st.metric("Saved vs old cost", f"AED {cost_saved:,}")

st.divider()

# ═══════════════════════════════════════════════════════════════════════════════
# ROW 6 — Scrape Log
# ═══════════════════════════════════════════════════════════════════════════════
with st.expander("🔍 Scrape history"):
    if scrape_log:
        df_log = pd.DataFrame(scrape_log)
        st.dataframe(df_log[["ran_at", "prs_found", "pods_found", "status", "notes"]],
                     use_container_width=True)
    else:
        st.info("No scrape runs recorded yet.")

st.caption("DEWA PR Tracker · Built with Playwright + SQLite + Streamlit")

+        "Language", ["All", "English (EN)", "Arabic (AR)"],

+    if pod_lang_filter == "English (EN)":

+        df_pod_show = df_pod_show[df_pod_show["language"] == "en"]

+    elif pod_lang_filter == "Arabic (AR)":

+        df_pod_show = df_pod_show[df_pod_show["language"] == "ar"]

+    pod_search = st.text_input("Search podcasts", placeholder="Filter by title…")

+        df_pod_show = df_pod_show[df_pod_show["title"].str.contains(pod_search, case=False, na=False)]

+    st.caption(f"Showing {len(df_pod_show)} of {len(df_pod)} episodes")

+    for _, pod in df_pod_show.head(100).iterrows():


Bash

source venv/bin/activate && streamlit run dashboard.py --server.headless true 2>&1 &
sleep 4 && curl -s -o /dev/null -w "%{http_code}" http://localhost:8501

Start Streamlit dashboard


Collecting usage statistics. To deactivate, set browser.gatherUsageStats to false.

/Users/khalidbinkalban/Projects/dewa-pr-tracker/venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
  warnings.warn(

  You can now view your Streamlit app in your browser.

  Local URL: http://localhost:8501
  Network URL: http://192.168.1.140:8501
  External URL: http://86.99.11.212:8501

  For better performance, install the Watchdog module:

  $ xcode-select --install
  $ pip install watchdog

200


Dashboard is live. Let me write a simple launch script so you can start everything with one command:


Write

#!/bin/bash
# DEWA PR Tracker — Start dashboard
cd "$(dirname "$0")"
source venv/bin/activate
echo "Starting DEWA PR Tracker dashboard..."
open http://localhost:8501
streamlit run dashboard.py


File created successfully at: /Users/khalidbinkalban/Projects/dewa-pr-tracker/start.sh

#!/bin/bash
# DEWA PR Tracker — Start dashboard
cd "$(dirname "$0")"
source venv/bin/activate
echo "Starting DEWA PR Tracker dashboard..."
open http://localhost:8501
streamlit run dashboard.py


Bash

chmod +x /Users/khalidbinkalban/Projects/dewa-pr-tracker/start.sh && ls -la /Users/khalidbinkalban/Projects/dewa-pr-tracker/

Make start script executable and list project files


total 15664
drwxr-xr-x@ 18 khalidbinkalban  staff      576 Apr  4 08:53 .
drwxr-xr-x@  4 khalidbinkalban  staff      128 Apr  4 08:22 ..
-rw-r--r--@  1 khalidbinkalban  staff     1876 Apr  4 08:22 config.py
-rw-r--r--@  1 khalidbinkalban  staff    17132 Apr  4 08:47 dashboard.py
-rw-r--r--@  1 khalidbinkalban  staff     6760 Apr  4 08:47 database.py
-rw-r--r--@  1 khalidbinkalban  staff    18393 Apr  4 08:37 debug_listingv1_response.html
-rw-r--r--@  1 khalidbinkalban  staff   295390 Apr  4 08:26 debug_news.html
-rw-r--r--@  1 khalidbinkalban  staff   781420 Apr  4 08:26 debug_news.png
-rw-r--r--@  1 khalidbinkalban  staff   178227 Apr  4 08:26 debug_podcasts.html
-rw-r--r--@  1 khalidbinkalban  staff    88389 Apr  4 08:26 debug_podcasts.png
-rw-r--r--@  1 khalidbinkalban  staff   279257 Apr  4 08:31 debug_pr_detail.html
-rw-r--r--@  1 khalidbinkalban  staff  4138698 Apr  4 08:32 debug_rss_feed.html
-rw-r--r--@  1 khalidbinkalban  staff  1449984 Apr  4 08:46 dewa_tracker.db
-rw-r--r--@  1 khalidbinkalban  staff       86 Apr  4 08:24 requirements.txt
-rw-r--r--@  1 khalidbinkalban  staff      850 Apr  4 08:24 scheduler.py
-rw-r--r--@  1 khalidbinkalban  staff    13927 Apr  4 08:47 scraper.py
-rwxr-xr-x@  1 khalidbinkalban  staff      195 Apr  4 08:53 start.sh
drwxr-xr-x@  8 khalidbinkalban  staff      256 Apr  4 08:25 venv


Clean up debug files


(Bash completed with no output)


Everything is running. Here's a full summary:

---

## What was built

**Location:** `~/Projects/dewa-pr-tracker/`

### Files
| File | Purpose |
|---|---|
| `scraper.py` | Playwright scraper (DEWA news) + RSS feed parser (rss.com) |
| `database.py` | SQLite layer — all queries and upserts |
| `dashboard.py` | Streamlit dashboard |
| `scheduler.py` | Daily auto-scrape at 08:00 Dubai time |
| `config.py` | All settings in one place |
| `start.sh` | One-command launcher |

### What it scraped (first run)
- **1,117 press releases** (2019–2026, all paginated)
- **691 podcast episodes** from `media.rss.com/dewa-news/feed.xml`
- **342 podcasts linked** to press releases by title matching

### Dashboard panels
1. **5 KPI cards** — total PRs, total podcasts, published today/this week/this month
2. **ROI Calculator** — adjustable podcast count & months. Currently: AED **0 spent** vs old cost of **20,000 AED × N podcasts**. Savings shown with a live gauge
3. **Trend charts** — daily / weekly / monthly publication counts
4. **Full PR table** — searchable, paginated, shows podcast count per PR
5. **Podcast player** — EN/AR filter, search, inline audio player + saved cost per episode
6. **Scrape log** — history of every run

### How to use

```bash
# Start dashboard
cd ~/Projects/dewa-pr-tracker
./start.sh

# Refresh data manually
cd ~/Projects/dewa-pr-tracker && source venv/bin/activate && python scraper.py

# Auto-refresh daily at 08:00 Dubai time (run in background)
source venv/bin/activate && python scheduler.py &
```

Dashboard is live now at **http://localhost:8501**


how many line of codes in this project so far?


find /Users/khalidbinkalban/Projects/dewa-pr-tracker -name "*.py" -o -name "*.sh" | grep -v venv | xargs wc -l

Count lines of code in project files


     391 /Users/khalidbinkalban/Projects/dewa-pr-tracker/scraper.py
      25 /Users/khalidbinkalban/Projects/dewa-pr-tracker/config.py
     197 /Users/khalidbinkalban/Projects/dewa-pr-tracker/database.py
     394 /Users/khalidbinkalban/Projects/dewa-pr-tracker/dashboard.py
      28 /Users/khalidbinkalban/Projects/dewa-pr-tracker/scheduler.py
       7 /Users/khalidbinkalban/Projects/dewa-pr-tracker/start.sh
    1042 total


**1,042 lines** across 6 files:

| File | Lines |
|---|---|
| `dashboard.py` | 394 |
| `scraper.py` | 391 |
| `database.py` | 197 |
| `scheduler.py` | 28 |
| `config.py` | 25 |
| `start.sh` | 7 |
| **Total** | **1,042** |


total lines?


**1,042 lines** — that's the total shown at the bottom of the output.


[Image #1]
- not working
- always use SVG icones not emojies


[Image: source: /var/folders/vn/5x83t3r1077gs8kcsydcn2fw0000gn/T/TemporaryItems/NSIRD_screencaptureui_Bd4S0U/Screenshot 2026-04-04 at 8.55.45 AM.png]