updating meetup ingestion to better handle location data

src/django_project/tests/unit/web/test_tasks.py

@@ -12,7 +12,8 @@
 django.setup()
 from model_bakery import baker
 from web.models import Event
-from web.tasks import ingest_future_eventbrite_events, post_event_to_linkedin
+from web.tasks import ingest_future_eventbrite_events, ingest_future_meetup_events, post_event_to_linkedin
+from web.utilities.scrapers.meetup import get_event_information
 
 
 class TestIngestFutureEventbriteEvents(TestCase):
@@ -109,6 +110,76 @@ def test_truncates_eventbrite_fields_to_model_limits(self, mock_get_events_for_o
         self.assertEqual(event.location_address, long_location_address[:256])
 
 
+class TestMeetupEventInformation(TestCase):
+    @patch("web.utilities.scrapers.meetup.fetch_content_with_playwright")
+    def test_reads_venue_when_meetup_json_field_order_changes(self, mock_fetch_content):
+        mock_fetch_content.return_value = """
+            <script type="application/json">
+              {"event":{"venue":{"country":"us","city":"Spokane","__typename":"Venue",
+              "name":"Startup Spokane","state":"WA","address":"25 W Main Ave","id":"123"}}}
+            </script>
+        """
+
+        event = get_event_information("https://www.meetup.com/example/events/123456/")
+
+        self.assertEqual(event["location_name"], "Startup Spokane")
+        self.assertEqual(event["location_address"], "25 W Main Ave, Spokane, WA, US")
+
+    @patch("web.utilities.scrapers.meetup.fetch_content_with_playwright")
+    def test_leaves_missing_or_tbd_venue_blank(self, mock_fetch_content):
+        mock_fetch_content.return_value = """
+            <script type="application/json">
+              {"event":{"venue":{"__typename":"Venue","name":"TBD","address":"123 Main St",
+              "city":"Spokane","state":"WA","country":"US"}}}
+            </script>
+        """
+
+        event = get_event_information("https://www.meetup.com/example/events/123456/")
+
+        self.assertEqual(event["location_name"], "")
+        self.assertEqual(event["location_address"], "")
+
+    @patch("web.utilities.scrapers.meetup.fetch_content_with_playwright")
+    def test_leaves_location_blank_when_venue_is_absent(self, mock_fetch_content):
+        mock_fetch_content.return_value = '<script type="application/json">{"event":{}}</script>'
+
+        event = get_event_information("https://www.meetup.com/example/events/123456/")
+
+        self.assertEqual(event["location_name"], "")
+        self.assertEqual(event["location_address"], "")
+
+
+class TestIngestFutureMeetupEvents(TestCase):
+    def setUp(self):
+        self.platform = baker.make("web.SocialPlatform", name="Meetup")
+        self.group = baker.make("web.TechGroup", name="Test Meetup Group", platform=self.platform)
+        self.link = baker.make(
+            "web.Link",
+            name=f"{self.group.name} {self.group.platform.name} page",
+            url="https://www.meetup.com/test-meetup-group",
+        )
+        self.group.links.add(self.link)
+
+    @patch("web.tasks.get_event_information")
+    @patch("web.tasks.get_event_links")
+    def test_truncates_venue_fields_before_saving(self, mock_get_event_links, mock_get_event_information):
+        mock_get_event_links.return_value = ["https://www.meetup.com/test-meetup-group/events/123456/"]
+        mock_get_event_information.return_value = {
+            "name": "Test event",
+            "url": "https://www.meetup.com/test-meetup-group/events/123456/",
+            "social_platform_id": "123456",
+            "start_datetime": "2026-09-20T18:00:00Z",
+            "location_name": "N" * 65,
+            "location_address": "A" * 257,
+        }
+
+        ingest_future_meetup_events(self.group.pk)
+
+        event = Event.objects.get(social_platform_id="123456")
+        self.assertEqual(event.location_name, "N" * 64)
+        self.assertEqual(event.location_address, "A" * 256)
+
+
 class TestPostEventToLinkedIn(TestCase):
     def test_skips_when_post_to_linkedin_setting_is_false(self):
         event = baker.make("web.Event")

src/django_project/web/tasks.py

@@ -116,6 +116,10 @@ def ingest_future_meetup_events(group_pk) -> str:
                 event_info.setdefault("location_name", "")
                 event_info.setdefault("location_address", "")
                 event_info.setdefault("map_link", "")
+                event_info["location_name"] = _truncate_for_model(Event, "location_name", event_info["location_name"])
+                event_info["location_address"] = _truncate_for_model(
+                    Event, "location_address", event_info["location_address"]
+                )
                 if not event_info.get("name", None):
                     logger.error("error parsing name for event hosted by %s; data = %s", group.name, event_info)
                     continue

src/django_project/web/utilities/scrapers/meetup.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import html
+import json
 import re
 from datetime import datetime, timedelta, timezone
 from typing import Any
@@ -10,6 +11,60 @@
 
 from web.utilities.html_utils import fetch_content, fetch_content_with_playwright
 
+_TBD_LOCATION_NAMES = {"tbd", "tdb", "to be determined"}
+
+
+def _iter_mappings(value: Any):
+    """Yield every dictionary contained in a decoded Meetup data payload."""
+    if isinstance(value, dict):
+        yield value
+        for child in value.values():
+            yield from _iter_mappings(child)
+    elif isinstance(value, list):
+        for child in value:
+            yield from _iter_mappings(child)
+
+
+def _get_venue(page_content: str, soup: BeautifulSoup) -> dict[str, Any] | None:
+    """Find Meetup's Venue object without depending on JSON field order."""
+    for script in soup.find_all("script"):
+        if not script.string:
+            continue
+        try:
+            payload = json.loads(script.string)
+        except json.JSONDecodeError:
+            continue
+        for item in _iter_mappings(payload):
+            if item.get("__typename") == "Venue":
+                return item
+
+    # Meetup sometimes embeds serialized JSON in its page data rather than a
+    # standalone JSON script. Decode the Venue object when it is available.
+    venue_match = re.search(r'\{[^{}]*"__typename"\s*:\s*"Venue"[^{}]*\}', page_content)
+    if venue_match:
+        try:
+            return json.loads(venue_match.group())
+        except json.JSONDecodeError:
+            pass
+    return None
+
+
+def _format_venue_address(venue: dict[str, Any]) -> str:
+    """Return the display address supplied by Meetup, or an empty string."""
+    address = venue.get("address")
+    if isinstance(address, dict):
+        return str(address.get("localized_address_display") or address.get("display") or "")
+    if address:
+        country = venue.get("country")
+        parts = [
+            address,
+            venue.get("city"),
+            venue.get("state"),
+            country.upper() if isinstance(country, str) else country,
+        ]
+        return ", ".join(str(part) for part in parts if part)
+    return ""
+
 
 def get_end_datetime(datetime_string: str, time_string: str) -> datetime | None:
     """create a datetime object with timezone information from information parsed from a meetup.com event page
@@ -142,27 +197,25 @@ def get_event_information(url: str) -> dict:
                     else:
                         event_info["end_datetime"] = None
 
-        location_name: str | Any = None
-        match = re.search(r'"__typename":"Venue","id":"\d+","name":"([^"]+)"', page_content)
-        if match:
-            location_name = match.group(1)
-        if not location_name:
+        venue = _get_venue(page_content, soup)
+        location_name: str = ""
+        location_address: str = ""
+        is_tbd_location = False
+        if venue:
+            location_name = str(venue.get("name") or "").strip()
+            location_address = _format_venue_address(venue)
+
+            # Meetup uses TBD for events whose physical location is not yet
+            # announced. Treat it as no location rather than displaying it.
+            if location_name.casefold() in _TBD_LOCATION_NAMES:
+                is_tbd_location = True
+                location_name = ""
+                location_address = ""
+        if not location_name and not is_tbd_location:
             online_p = soup.find("p", class_="ds2-k16 text-ds2-text-fill-primary-enabled")
             if online_p and online_p.get_text(strip=True) == "Online event":
                 location_name = "Online event"
         event_info["location_name"] = location_name
-
-        location_address: str = ""
-        address_match: re.Match[str] | None = re.search(
-            r'"__typename":"Venue","id":"\d+","name":"[^"]+","address":"([^"]+)","city":"([^"]+)","state":"([^"]+)","country":"([^"]+)"',
-            page_content,
-        )
-        if address_match:
-            street: str | Any = address_match.group(1)
-            city: str | Any = address_match.group(2)
-            state: str | Any = address_match.group(3)
-            country: str | Any = address_match.group(4)
-            location_address = f"{street}, {city}, {state}, {country.upper()}"
         event_info["location_address"] = location_address
 
         map_link: str = ""