From fe5f1cc7bd97ad24b6586429f09f786ebf020f5b Mon Sep 17 00:00:00 2001 From: David Slusser Date: Thu, 6 Aug 2026 12:31:12 -0700 Subject: [PATCH 1/3] updating version for LinkedIn posts --- .../tests/unit/web/test_linkedin_notifier.py | 32 +++++++++++- .../web/utilities/notifiers/linkedin.py | 51 +++++++++++++++---- 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/src/django_project/tests/unit/web/test_linkedin_notifier.py b/src/django_project/tests/unit/web/test_linkedin_notifier.py index 59e421d..bb04bb8 100644 --- a/src/django_project/tests/unit/web/test_linkedin_notifier.py +++ b/src/django_project/tests/unit/web/test_linkedin_notifier.py @@ -114,7 +114,7 @@ def test_uses_configured_api_version_header(self): self.assertEqual(client.headers["LinkedIn-Version"], "202607") def test_defaults_api_version_header_to_current_year_month(self): - frozen_now = datetime(2026, 7, 22, tzinfo=UTC) + frozen_now = datetime(2026, 8, 6, tzinfo=UTC) with ( patch("web.utilities.notifiers.linkedin.settings.LINKEDIN_API_VERSION", "", create=True), @@ -124,6 +124,36 @@ def test_defaults_api_version_header_to_current_year_month(self): self.assertEqual(client.headers["LinkedIn-Version"], "202607") + def test_post_retries_once_after_default_version_426(self): + future_version_response = Mock(status_code=426) + future_version_response.raise_for_status.side_effect = requests.HTTPError(response=future_version_response) # type: ignore[name-defined] + future_version_response.json.return_value = { + "error": "Unsupported API version", + "message": "The API version you are using has been sunset. Please upgrade to a supported version.", + } + + success_response = Mock(status_code=201) + success_response.raise_for_status.return_value = None + + frozen_now = datetime(2026, 8, 6, tzinfo=UTC) + + with ( + patch("web.utilities.notifiers.linkedin.settings.LINKEDIN_API_VERSION", "", create=True), + patch("web.utilities.notifiers.linkedin.timezone.now", return_value=frozen_now), + patch( + "web.utilities.notifiers.linkedin.requests.post", + side_effect=[future_version_response, success_response], + ) as mock_post, + ): + client = self.build_client() + client.api_version = "202608" + client.set_headers() + response = client.post_organization_post("hello world") + + self.assertIs(response, success_response) + self.assertEqual(client.headers["LinkedIn-Version"], "202607") + self.assertEqual(mock_post.call_count, 2) + def test_refresh_access_token_updates_db_credential_when_present(self): credential = DummyCredential() credential.__class__.objects = DummyCredentialManager(credential) diff --git a/src/django_project/web/utilities/notifiers/linkedin.py b/src/django_project/web/utilities/notifiers/linkedin.py index fefacc0..2d1c31d 100644 --- a/src/django_project/web/utilities/notifiers/linkedin.py +++ b/src/django_project/web/utilities/notifiers/linkedin.py @@ -42,14 +42,18 @@ def __init__( self.refresh_token = refresh_token self.env_path = Path(env_path) if env_path else None self.credential = credential - self.api_version = ( - api_version or getattr(settings, "LINKEDIN_API_VERSION", "") or timezone.now().strftime("%Y%m") - ) + configured_api_version = api_version or getattr(settings, "LINKEDIN_API_VERSION", "") + self.api_version = configured_api_version or self._default_api_version() + self.uses_default_api_version = not configured_api_version self.post_url = "https://api.linkedin.com/rest/posts" self.access_token_url = "https://www.linkedin.com/oauth/v2/accessToken" # nosec B105 self.authorization_url = "https://www.linkedin.com/oauth/v2/authorization" self.set_headers() + def _default_api_version(self) -> str: + previous_month = (timezone.now().replace(day=1) - timedelta(days=1)).strftime("%Y%m") + return previous_month + def set_headers(self) -> None: self.headers: dict[str, str] = { "Authorization": f"Bearer {self.access_token}", @@ -58,6 +62,19 @@ def set_headers(self) -> None: "X-Restli-Protocol-Version": "2.0.0", } + def _is_version_failure(self, response: Optional[requests.Response]) -> bool: + if response is None or response.status_code != 426: + return False + try: + response_json = response.json() + except ValueError: + return False + message_parts = [ + str(response_json.get("message", "")), + str(response_json.get("error", "")), + ] + return "version" in " ".join(message_parts).lower() + def can_refresh_access_token(self) -> bool: return bool(self.refresh_token and self.client_id and self.client_secret) @@ -246,14 +263,28 @@ def post_organization_post( response.raise_for_status() return response except requests.HTTPError: - if not self._is_auth_failure(response) or not self.can_refresh_access_token(): - raise + if self._is_auth_failure(response) and self.can_refresh_access_token(): + logger.info("LinkedIn post received %s; refreshing access token and retrying once.", response.status_code) + self.refresh_access_token() + retry_response = requests.post(self.post_url, headers=self.headers, data=payload_json, timeout=15) + retry_response.raise_for_status() + return retry_response + + if self.uses_default_api_version and self._is_version_failure(response): + fallback_version = self._default_api_version() + if fallback_version != self.api_version: + logger.info( + "LinkedIn post received 426 for version %s; retrying once with fallback version %s.", + self.api_version, + fallback_version, + ) + self.api_version = fallback_version + self.set_headers() + retry_response = requests.post(self.post_url, headers=self.headers, data=payload_json, timeout=15) + retry_response.raise_for_status() + return retry_response - logger.info("LinkedIn post received %s; refreshing access token and retrying once.", response.status_code) - self.refresh_access_token() - retry_response = requests.post(self.post_url, headers=self.headers, data=payload_json, timeout=15) - retry_response.raise_for_status() - return retry_response + raise def build_event_commentary( self, From 701983700da965b94d26348705d19569a271b415 Mon Sep 17 00:00:00 2001 From: David Slusser Date: Thu, 6 Aug 2026 12:41:45 -0700 Subject: [PATCH 2/3] updates per PR comments --- .../tests/unit/web/test_linkedin_notifier.py | 30 ++++++++++---- .../web/utilities/notifiers/linkedin.py | 40 +++++++++++-------- 2 files changed, 47 insertions(+), 23 deletions(-) diff --git a/src/django_project/tests/unit/web/test_linkedin_notifier.py b/src/django_project/tests/unit/web/test_linkedin_notifier.py index bb04bb8..5454902 100644 --- a/src/django_project/tests/unit/web/test_linkedin_notifier.py +++ b/src/django_project/tests/unit/web/test_linkedin_notifier.py @@ -107,6 +107,22 @@ def test_post_retries_once_after_auth_failure(self): self.assertEqual(client.access_token, "new-token") self.assertEqual(mock_post.call_count, 3) + def test_post_does_not_retry_after_forbidden_response(self): + client = self.build_client() + + forbidden_response = Mock(status_code=403) + forbidden_response.raise_for_status.side_effect = requests.HTTPError(response=forbidden_response) # type: ignore[name-defined] + + with ( + patch("web.utilities.notifiers.linkedin.requests.post", return_value=forbidden_response) as mock_post, + patch("web.utilities.notifiers.linkedin.settings"), + self.assertRaises(requests.HTTPError), + ): + client.post_organization_post("hello world") + + self.assertEqual(client.access_token, "old-token") + self.assertEqual(mock_post.call_count, 1) + def test_uses_configured_api_version_header(self): with patch("web.utilities.notifiers.linkedin.settings.LINKEDIN_API_VERSION", "202607", create=True): client = self.build_client() @@ -125,9 +141,11 @@ def test_defaults_api_version_header_to_current_year_month(self): self.assertEqual(client.headers["LinkedIn-Version"], "202607") def test_post_retries_once_after_default_version_426(self): - future_version_response = Mock(status_code=426) - future_version_response.raise_for_status.side_effect = requests.HTTPError(response=future_version_response) # type: ignore[name-defined] - future_version_response.json.return_value = { + unsupported_version_response = Mock(status_code=426) + unsupported_version_response.raise_for_status.side_effect = requests.HTTPError( # type: ignore[name-defined] + response=unsupported_version_response + ) + unsupported_version_response.json.return_value = { "error": "Unsupported API version", "message": "The API version you are using has been sunset. Please upgrade to a supported version.", } @@ -142,16 +160,14 @@ def test_post_retries_once_after_default_version_426(self): patch("web.utilities.notifiers.linkedin.timezone.now", return_value=frozen_now), patch( "web.utilities.notifiers.linkedin.requests.post", - side_effect=[future_version_response, success_response], + side_effect=[unsupported_version_response, success_response], ) as mock_post, ): client = self.build_client() - client.api_version = "202608" - client.set_headers() response = client.post_organization_post("hello world") self.assertIs(response, success_response) - self.assertEqual(client.headers["LinkedIn-Version"], "202607") + self.assertEqual(client.headers["LinkedIn-Version"], "202606") self.assertEqual(mock_post.call_count, 2) def test_refresh_access_token_updates_db_credential_when_present(self): diff --git a/src/django_project/web/utilities/notifiers/linkedin.py b/src/django_project/web/utilities/notifiers/linkedin.py index 2d1c31d..1f2fde1 100644 --- a/src/django_project/web/utilities/notifiers/linkedin.py +++ b/src/django_project/web/utilities/notifiers/linkedin.py @@ -54,6 +54,13 @@ def _default_api_version(self) -> str: previous_month = (timezone.now().replace(day=1) - timedelta(days=1)).strftime("%Y%m") return previous_month + def _previous_api_version(self, api_version: str) -> str: + year = int(api_version[:4]) + month = int(api_version[4:]) + if month == 1: + return f"{year - 1}12" + return f"{year}{month - 1:02d}" + def set_headers(self) -> None: self.headers: dict[str, str] = { "Authorization": f"Bearer {self.access_token}", @@ -226,10 +233,10 @@ def _persist_tokens(self, token_data: Optional[dict[str, Any]] = None) -> None: self.env_path.write_text("\n".join(lines) + "\n", encoding="utf-8") - def _is_auth_failure(self, response: Optional[requests.Response]) -> bool: + def _is_retryable_token_failure(self, response: Optional[requests.Response]) -> bool: if response is None: return False - return response.status_code in {401, 403} + return response.status_code == 401 def post_organization_post( self, @@ -263,26 +270,27 @@ def post_organization_post( response.raise_for_status() return response except requests.HTTPError: - if self._is_auth_failure(response) and self.can_refresh_access_token(): - logger.info("LinkedIn post received %s; refreshing access token and retrying once.", response.status_code) + if self._is_retryable_token_failure(response) and self.can_refresh_access_token(): + logger.info( + "LinkedIn post received %s; refreshing access token and retrying once.", response.status_code + ) self.refresh_access_token() retry_response = requests.post(self.post_url, headers=self.headers, data=payload_json, timeout=15) retry_response.raise_for_status() return retry_response if self.uses_default_api_version and self._is_version_failure(response): - fallback_version = self._default_api_version() - if fallback_version != self.api_version: - logger.info( - "LinkedIn post received 426 for version %s; retrying once with fallback version %s.", - self.api_version, - fallback_version, - ) - self.api_version = fallback_version - self.set_headers() - retry_response = requests.post(self.post_url, headers=self.headers, data=payload_json, timeout=15) - retry_response.raise_for_status() - return retry_response + fallback_version = self._previous_api_version(self.api_version) + logger.info( + "LinkedIn post received 426 for version %s; retrying once with fallback version %s.", + self.api_version, + fallback_version, + ) + self.api_version = fallback_version + self.set_headers() + retry_response = requests.post(self.post_url, headers=self.headers, data=payload_json, timeout=15) + retry_response.raise_for_status() + return retry_response raise From f5835fca940441a98cdb8cf783086b574d966642 Mon Sep 17 00:00:00 2001 From: David Slusser Date: Sun, 9 Aug 2026 21:00:26 -0700 Subject: [PATCH 3/3] ruff fixes; remove isort --- .github/hooks/pre-commit | 1 - .github/workflows/isort.yaml | 18 --- src/django_project/blogs/admin.py | 31 +++-- src/django_project/blogs/models.py | 2 +- src/django_project/blogs/urls.py | 5 +- src/django_project/blogs/views.py | 3 +- src/django_project/core/celery.py | 2 +- src/django_project/core/settings.py | 2 + src/django_project/core/urls.py | 3 +- src/django_project/web/admin.py | 36 ++--- .../web/management/commands/linkedin_oauth.py | 1 + src/django_project/web/models.py | 10 +- src/django_project/web/tasks.py | 16 ++- src/django_project/web/urls.py | 5 +- src/django_project/web/utilities/dt_utils.py | 3 +- .../web/utilities/html_utils.py | 20 ++- .../web/utilities/notifiers/discord.py | 3 + .../web/utilities/notifiers/linkedin.py | 43 +++--- .../web/utilities/scrapers/eventbrite.py | 13 +- .../web/utilities/scrapers/meetup.py | 127 +++++++++--------- src/django_project/web/views.py | 1 + 21 files changed, 178 insertions(+), 167 deletions(-) delete mode 100644 .github/workflows/isort.yaml diff --git a/.github/hooks/pre-commit b/.github/hooks/pre-commit index b3765b8..766ee87 100755 --- a/.github/hooks/pre-commit +++ b/.github/hooks/pre-commit @@ -25,7 +25,6 @@ function run_checks { # add pre-commit commands here run_checks "bandit" bandit pyproject.toml -r -run_checks "isort" isort src --check run_checks "mypy" mypy src run_checks "ruff check" ruff check src run_checks "ruff format" ruff format src diff --git a/.github/workflows/isort.yaml b/.github/workflows/isort.yaml deleted file mode 100644 index 5b52f95..0000000 --- a/.github/workflows/isort.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: Import Order Validation (isort) - -on: - push: - branches-ignore: - - main - -jobs: - isort: - runs-on: ubuntu-latest - name: "isort" - if: github.event.created == false # Skip if this push created a new branch - steps: - - uses: davidslusser/actions_python_isort@v1.0.1 - with: - src: "src/django_project" - options: "--check --diff" - python_version: "3.13" diff --git a/src/django_project/blogs/admin.py b/src/django_project/blogs/admin.py index 642e20a..6ea4b1e 100644 --- a/src/django_project/blogs/admin.py +++ b/src/django_project/blogs/admin.py @@ -1,22 +1,25 @@ # import models -from blogs.models import BlogPlatform, BlogPost, BlogSeries, BlogTag +from __future__ import annotations + from django.contrib import admin +from blogs.models import BlogPlatform, BlogPost, BlogSeries, BlogTag + class BlogPlatformAdmin(admin.ModelAdmin): - list_display: list[str] = ["id", "created_at", "updated_at", "enabled", "name", "website_url"] - search_fields: list[str] = ["id", "name", "website_url"] - list_filter: list[str] = ["enabled"] + list_display: tuple[str, ...] = ("id", "created_at", "updated_at", "enabled", "name", "website_url") + search_fields: tuple[str, ...] = ("id", "name", "website_url") + list_filter: tuple[str, ...] = ("enabled",) class BlogSeriesAdmin(admin.ModelAdmin): - list_display: list[str] = ["id", "created_at", "updated_at", "name", "description"] - search_fields: list[str] = ["id", "name", "description"] - list_filter: list = [] + list_display: tuple[str, ...] = ("id", "created_at", "updated_at", "name", "description") + search_fields: tuple[str, ...] = ("id", "name", "description") + list_filter: tuple[()] = () class BlogPostAdmin(admin.ModelAdmin): - list_display: list[str] = [ + list_display: tuple[str, ...] = ( "id", "created_at", "updated_at", @@ -27,15 +30,15 @@ class BlogPostAdmin(admin.ModelAdmin): "image", "author", "series", - ] - search_fields: list[str] = ["id", "title", "description", "url", "image", "author"] - list_filter: list[str] = ["platform", "series"] + ) + search_fields: tuple[str, ...] = ("id", "title", "description", "url", "image", "author") + list_filter: tuple[str, ...] = ("platform", "series") class BlogTagAdmin(admin.ModelAdmin): - list_display: list[str] = ["id", "created_at", "updated_at", "value"] - search_fields: list[str] = ["id", "value"] - list_filter: list = [] + list_display: tuple[str, ...] = ("id", "created_at", "updated_at", "value") + search_fields: tuple[str, ...] = ("id", "value") + list_filter: tuple[()] = () # register models diff --git a/src/django_project/blogs/models.py b/src/django_project/blogs/models.py index 0fbbce8..901f23c 100644 --- a/src/django_project/blogs/models.py +++ b/src/django_project/blogs/models.py @@ -65,7 +65,7 @@ class BlogTag(HandyHelperBaseModel): value = models.CharField(max_length=64, unique=True, null=False) class Meta: - ordering = ["value"] + ordering = ("value",) def __str__(self) -> str: return self.value diff --git a/src/django_project/blogs/urls.py b/src/django_project/blogs/urls.py index 59b73fb..13e2555 100644 --- a/src/django_project/blogs/urls.py +++ b/src/django_project/blogs/urls.py @@ -1,7 +1,10 @@ -from blogs import views +from __future__ import annotations + from django.urls import path from django.urls.resolvers import URLPattern +from blogs import views + app_name = "blogs" diff --git a/src/django_project/blogs/views.py b/src/django_project/blogs/views.py index d2ca060..930e40a 100644 --- a/src/django_project/blogs/views.py +++ b/src/django_project/blogs/views.py @@ -1,9 +1,10 @@ -from blogs.models import BlogPost from django.db.models.manager import BaseManager from django.http import HttpResponse from django.shortcuts import render from django.views import View +from blogs.models import BlogPost + class BlogPostListView(View): def get(self, request) -> HttpResponse: diff --git a/src/django_project/core/celery.py b/src/django_project/core/celery.py index 148c756..6a189ee 100644 --- a/src/django_project/core/celery.py +++ b/src/django_project/core/celery.py @@ -21,4 +21,4 @@ @celery_app.task(bind=True) def debug_task(self, *args, **kwargs): - print("Request: {0!r}".format(self.request)) + print(f"Request: {self.request!r}") diff --git a/src/django_project/core/settings.py b/src/django_project/core/settings.py index 60b4ad1..1a1ff2f 100644 --- a/src/django_project/core/settings.py +++ b/src/django_project/core/settings.py @@ -10,6 +10,8 @@ https://docs.djangoproject.com/en/4.2/ref/settings/ """ +from __future__ import annotations + import os import sys from pathlib import Path diff --git a/src/django_project/core/urls.py b/src/django_project/core/urls.py index 715ce2d..e9e8e5d 100644 --- a/src/django_project/core/urls.py +++ b/src/django_project/core/urls.py @@ -14,12 +14,13 @@ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ -from core.views import HostView, robots_txt from django.conf import settings from django.contrib import admin from django.urls import include, path from web.views import linkedin_oauth_callback +from core.views import HostView, robots_txt + urlpatterns: list = [ # Django provided URLs path("console/", admin.site.urls), diff --git a/src/django_project/web/admin.py b/src/django_project/web/admin.py index f9e6cf4..89a5063 100644 --- a/src/django_project/web/admin.py +++ b/src/django_project/web/admin.py @@ -12,41 +12,41 @@ class TagAdmin(admin.ModelAdmin): - list_display = ["id", "value", "created_at", "updated_at"] - search_fields = ["id", "value"] + list_display = ("id", "value", "created_at", "updated_at") + search_fields = ("id", "value") class LinkAdmin(admin.ModelAdmin): - list_display = ["id", "name", "description", "url", "created_at", "updated_at"] - search_fields = ["id", "name", "description", "url"] + list_display = ("id", "name", "description", "url", "created_at", "updated_at") + search_fields = ("id", "name", "description", "url") class SocialPlatformAdmin(admin.ModelAdmin): - list_display = ["id", "name", "enabled", "base_url", "created_at", "updated_at"] - search_fields = ["id", "name", "base_url"] - list_filter = ["enabled"] + list_display = ("id", "name", "enabled", "base_url", "created_at", "updated_at") + search_fields = ("id", "name", "base_url") + list_filter = ("enabled",) class IntegrationCredentialAdmin(admin.ModelAdmin): - list_display = [ + list_display = ( "id", "provider", "access_token_expires_at", "refresh_token_expires_at", "created_at", "updated_at", - ] - search_fields = ["id", "provider"] + ) + search_fields = ("id", "provider") class TechGroupAdmin(admin.ModelAdmin): - list_display = ["id", "name", "description", "enabled", "platform", "icon", "image", "created_at", "updated_at"] - search_fields = ["id", "name", "description", "icon", "image"] - list_filter = ["enabled", "platform"] + list_display = ("id", "name", "description", "enabled", "platform", "icon", "image", "created_at", "updated_at") + search_fields = ("id", "name", "description", "icon", "image") + list_filter = ("enabled", "platform") class EventAdmin(admin.ModelAdmin): - list_display = [ + list_display = ( "id", "name", "description", @@ -61,8 +61,8 @@ class EventAdmin(admin.ModelAdmin): "image", "created_at", "updated_at", - ] - search_fields = [ + ) + search_fields = ( "id", "name", "description", @@ -72,8 +72,8 @@ class EventAdmin(admin.ModelAdmin): "url", "social_platform_id", "image", - ] - list_filter = ["group"] + ) + list_filter = ("group",) # register models diff --git a/src/django_project/web/management/commands/linkedin_oauth.py b/src/django_project/web/management/commands/linkedin_oauth.py index 72cdcdc..6467768 100644 --- a/src/django_project/web/management/commands/linkedin_oauth.py +++ b/src/django_project/web/management/commands/linkedin_oauth.py @@ -1,6 +1,7 @@ from django.apps import apps from django.conf import settings from django.core.management.base import BaseCommand, CommandError + from web.utilities.notifiers.linkedin import LinkedInOrganizationClient diff --git a/src/django_project/web/models.py b/src/django_project/web/models.py index da386a4..ec721e8 100644 --- a/src/django_project/web/models.py +++ b/src/django_project/web/models.py @@ -48,7 +48,7 @@ class Event(HandyHelperBaseModel): image = models.ImageField(upload_to="tech_events/", blank=True, null=True) class Meta: - ordering = ["start_datetime"] + ordering = ("start_datetime",) def __str__(self) -> str: return self.name @@ -76,7 +76,7 @@ class SocialPlatform(HandyHelperBaseModel): base_url = models.URLField(blank=True, help_text="base url of provider") class Meta: - ordering = ["name"] + ordering = ("name",) def __str__(self) -> str: return self.name @@ -92,7 +92,7 @@ class IntegrationCredential(HandyHelperBaseModel): refresh_token_expires_at = models.DateTimeField(blank=True, null=True) class Meta: - ordering = ["provider"] + ordering = ("provider",) def __str__(self) -> str: return self.provider @@ -104,7 +104,7 @@ class Tag(HandyHelperBaseModel): value = models.CharField(max_length=64, unique=True, null=False) class Meta: - ordering = ["value"] + ordering = ("value",) def __str__(self) -> str: return self.value @@ -128,7 +128,7 @@ class TechGroup(HandyHelperBaseModel): discord_webhook_url = EncryptedTextField(blank=True, null=True) class Meta: - ordering = ["name"] + ordering = ("name",) def __str__(self) -> str: return self.name diff --git a/src/django_project/web/tasks.py b/src/django_project/web/tasks.py index 2fb43d7..2806dfc 100644 --- a/src/django_project/web/tasks.py +++ b/src/django_project/web/tasks.py @@ -11,6 +11,7 @@ from django.conf import settings from django.db.models.manager import BaseManager from django.utils import timezone + from web.models import Event, IntegrationCredential, Link, Tag, TechGroup from web.utilities.dt_utils import convert_to_pacific from web.utilities.notifiers.discord import DiscordNotifier @@ -27,12 +28,14 @@ get_group_description, ) +logger = logging.getLogger(__name__) + @shared_task(time_limit=30, max_retries=0, name="web.test_task") def test_task() -> str: - logging.info("test task starting") + logger.info("test task starting") time.sleep(3) - logging.info("test task completed") + logger.info("test task completed") return "test task completed!" @@ -62,10 +65,9 @@ def ingest_eventbrite_organization_details(group_pk) -> str: organization_details = get_organization_details(eb_group_id) description = organization_details["long_description"]["text"] - if description: - if group.description != description: - group.update(description=description) - updated = True + if description and group.description != description: + group.update(description=description) + updated = True if organization_details.get("website"): website = organization_details["website"] if website and not group.links.filter(url=website): @@ -104,7 +106,7 @@ def ingest_future_meetup_events(group_pk) -> str: event_info.setdefault("location_address", "") event_info.setdefault("map_link", "") if not event_info.get("name", None): - logging.error(f"error parsing name for event hosted by {group.name}; data = {event_info}") + logger.error("error parsing name for event hosted by %s; data = %s", group.name, event_info) continue if event_info["social_platform_id"]: _, is_new = Event.objects.update_or_create( diff --git a/src/django_project/web/urls.py b/src/django_project/web/urls.py index 105e359..fa4a058 100644 --- a/src/django_project/web/urls.py +++ b/src/django_project/web/urls.py @@ -1,4 +1,5 @@ from django.urls import path, re_path + from web import views app_name = "web" @@ -12,8 +13,8 @@ path("calendar/", views.EventCalendarView.as_view(), name="event_calendar"), path("events//", views.TechEventView.as_view(), name="get_event"), path("events//modal/", views.TechEventModalView.as_view(), name="techevent_modal"), - re_path("^events/(?P\w+)?$", views.TechEventsView.as_view(), name="get_events"), + re_path(r"^events/(?P\w+)?$", views.TechEventsView.as_view(), name="get_events"), path("techgroups//", views.TechGroupView.as_view(), name="get_techgroup"), path("techgroups//modal/", views.TechGroupModalView.as_view(), name="techgroup_modal"), - re_path("^techgroups/(?P\w+)?$", views.TechGroupsView.as_view(), name="get_techgroups"), + re_path(r"^techgroups/(?P\w+)?$", views.TechGroupsView.as_view(), name="get_techgroups"), ] diff --git a/src/django_project/web/utilities/dt_utils.py b/src/django_project/web/utilities/dt_utils.py index 099c125..b34c9c9 100644 --- a/src/django_project/web/utilities/dt_utils.py +++ b/src/django_project/web/utilities/dt_utils.py @@ -1,6 +1,5 @@ -from zoneinfo import ZoneInfo - from django.utils import timezone +from zoneinfo import ZoneInfo PACIFIC = ZoneInfo("America/Los_Angeles") diff --git a/src/django_project/web/utilities/html_utils.py b/src/django_project/web/utilities/html_utils.py index 72e92af..92b3f32 100644 --- a/src/django_project/web/utilities/html_utils.py +++ b/src/django_project/web/utilities/html_utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import time from typing import TYPE_CHECKING, Any @@ -9,6 +11,14 @@ from playwright.sync_api._generated import Browser, BrowserContext, Page +class FetchContentError(RuntimeError): + pass + + +class TargetNotFoundError(RuntimeError): + pass + + def fetch_content(url, timeout=30) -> bytes | Any: """fetch html content from a url using requests @@ -17,7 +27,7 @@ def fetch_content(url, timeout=30) -> bytes | Any: timeout (int, optional): timeout in seconds. Defaults to 30. Raises: - Exception: if the response status code is not 200 + FetchContentError: if the response status code is not 200 Returns: str: response text from the url @@ -27,7 +37,7 @@ def fetch_content(url, timeout=30) -> bytes | Any: if response.status_code == 200: return response.content else: - raise Exception(f"Failed to fetch content from {url}: {response.status_code}") + raise FetchContentError(f"Failed to fetch content from {url}: {response.status_code}") def fetch_content_with_playwright(url, retries=3, timeout=30000) -> str: @@ -57,7 +67,7 @@ def fetch_content_with_playwright(url, retries=3, timeout=30000) -> str: html_content: str = page.content() browser.close() return html_content - except Exception as e: + except (RuntimeError, requests.RequestException) as e: print(f"Error: {e}. Retrying... ({attempt + 1}/{retries})") attempt += 1 time.sleep(2 + attempt) @@ -77,7 +87,7 @@ def find_target( max_retries (int, optional): max count of retries to attempt. Defaults to 3. Raises: - Exception: if the target element is not found after max_retries + TargetNotFoundError: if the target element is not found after max_retries Returns: str: html content of the target element @@ -94,7 +104,7 @@ def find_target( print(f"Retry {retries}/{max_retries}: target_ul not found. Retrying in {1 + retries} seconds...") time.sleep(1 + retries) - raise Exception("target_ul not found after maximum retries") + raise TargetNotFoundError("target_ul not found after maximum retries") def convert_html_to_text(html_content: str) -> str: diff --git a/src/django_project/web/utilities/notifiers/discord.py b/src/django_project/web/utilities/notifiers/discord.py index 586cab3..1451ddc 100644 --- a/src/django_project/web/utilities/notifiers/discord.py +++ b/src/django_project/web/utilities/notifiers/discord.py @@ -1,9 +1,12 @@ +from __future__ import annotations + import json from multiprocessing.managers import BaseManager from typing import Any import requests from django.utils import timezone + from web.models import Event from web.utilities.ai.gemini import generate_post_content from web.utilities.ai.prompts import ( diff --git a/src/django_project/web/utilities/notifiers/linkedin.py b/src/django_project/web/utilities/notifiers/linkedin.py index 1f2fde1..bb19161 100644 --- a/src/django_project/web/utilities/notifiers/linkedin.py +++ b/src/django_project/web/utilities/notifiers/linkedin.py @@ -1,14 +1,17 @@ +from __future__ import annotations + import json import logging from datetime import timedelta from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any import requests from bs4 import BeautifulSoup from django.conf import settings from django.db import transaction from django.utils import timezone + from web.utilities.ai.gemini import generate_post_content from web.utilities.ai.prompts import ( create_event_reminder_prompt, @@ -26,14 +29,14 @@ class LinkedInOrganizationClient: def __init__( self, - access_token: Optional[str], + access_token: str | None, organization_urn: str, - client_id: Optional[str] = None, - client_secret: Optional[str] = None, - refresh_token: Optional[str] = None, - env_path: Optional[str] = None, - credential: Optional[Any] = None, - api_version: Optional[str] = None, + client_id: str | None = None, + client_secret: str | None = None, + refresh_token: str | None = None, + env_path: str | None = None, + credential: Any | None = None, + api_version: str | None = None, ) -> None: self.access_token = access_token self.organization_urn: str = organization_urn @@ -69,7 +72,7 @@ def set_headers(self) -> None: "X-Restli-Protocol-Version": "2.0.0", } - def _is_version_failure(self, response: Optional[requests.Response]) -> bool: + def _is_version_failure(self, response: requests.Response | None) -> bool: if response is None or response.status_code != 426: return False try: @@ -103,7 +106,7 @@ def refresh_access_token(self) -> None: self._apply_token_data(token_data) self._persist_tokens(token_data) - def build_authorization_url(self, redirect_uri: str, scope: str, state: Optional[str] = None) -> str: + def build_authorization_url(self, redirect_uri: str, scope: str, state: str | None = None) -> str: if not self.client_id: raise ValueError("LinkedIn client ID is required to build the authorization URL.") @@ -146,7 +149,7 @@ def exchange_authorization_code(self, code: str, redirect_uri: str) -> dict[str, self._persist_tokens(token_data) return token_data - def _request_token_refresh(self, refresh_token: Optional[str]) -> dict[str, Any]: + def _request_token_refresh(self, refresh_token: str | None) -> dict[str, Any]: response = requests.post( self.access_token_url, data={ @@ -182,9 +185,9 @@ def _refresh_access_token_with_credential(self) -> None: self.credential = locked_credential self._persist_tokens(token_data) - def _persist_tokens(self, token_data: Optional[dict[str, Any]] = None) -> None: - setattr(settings, "LINKEDIN_ACCESS_TOKEN", self.access_token) - setattr(settings, "LINKEDIN_REFRESH_TOKEN", self.refresh_token) + def _persist_tokens(self, token_data: dict[str, Any] | None = None) -> None: + settings.LINKEDIN_ACCESS_TOKEN = self.access_token + settings.LINKEDIN_REFRESH_TOKEN = self.refresh_token if self.credential is not None: self.credential.access_token = self.access_token @@ -233,7 +236,7 @@ def _persist_tokens(self, token_data: Optional[dict[str, Any]] = None) -> None: self.env_path.write_text("\n".join(lines) + "\n", encoding="utf-8") - def _is_retryable_token_failure(self, response: Optional[requests.Response]) -> bool: + def _is_retryable_token_failure(self, response: requests.Response | None) -> bool: if response is None: return False return response.status_code == 401 @@ -241,9 +244,9 @@ def _is_retryable_token_failure(self, response: Optional[requests.Response]) -> def post_organization_post( self, commentary: str, - article_url: Optional[str] = None, - article_title: Optional[str] = None, - article_description: Optional[str] = None, + article_url: str | None = None, + article_title: str | None = None, + article_description: str | None = None, ) -> requests.Response: self.ensure_access_token() payload: dict[str, Any] = { @@ -296,7 +299,7 @@ def post_organization_post( def build_event_commentary( self, - event: "Event", + event: Event, is_new: bool = True, ) -> str: """ @@ -343,7 +346,7 @@ def build_event_commentary( def post_event( self, - event: "Event", + event: Event, is_new: bool = True, ) -> requests.Response: """ diff --git a/src/django_project/web/utilities/scrapers/eventbrite.py b/src/django_project/web/utilities/scrapers/eventbrite.py index 5982b98..deca1ae 100644 --- a/src/django_project/web/utilities/scrapers/eventbrite.py +++ b/src/django_project/web/utilities/scrapers/eventbrite.py @@ -9,6 +9,10 @@ from requests.exceptions import HTTPError, RequestException +class EventbriteRateLimitError(RuntimeError): + pass + + def create_google_map_link(address: str) -> str: """create a link to a Google map for a provided address @@ -37,8 +41,7 @@ def filter_events_by_date(events: list, date_filter: timezone) -> list: """ filtered_events: list = [] for event in events: - created_date: datetime = datetime.strptime(event["created"], "%Y-%m-%dT%H:%M:%SZ") - created_date = timezone.make_aware(created_date, timezone.utc) + created_date = datetime.fromisoformat(event["created"].replace("Z", "+00:00")) if created_date > date_filter: filtered_events.append(event) return filtered_events @@ -144,7 +147,7 @@ def get_event_details(event_id: str) -> dict: continue # Skip the rest of the loop and retry else: print(f"Max retries ({MAX_RETRIES}) for 429 reached. Raising the last error.") - raise Exception(f"Max retries ({MAX_RETRIES})) reached for 429 Too Many Requests.") + raise EventbriteRateLimitError(f"Max retries ({MAX_RETRIES}) reached for 429 Too Many Requests.") # If not a 429, raise for other status codes resp.raise_for_status() @@ -161,7 +164,7 @@ def get_event_details(event_id: str) -> dict: time.sleep(sleep_for) else: print(f"Max retries ({MAX_RETRIES}) reached for non-429 HTTP error. Raising the last error.") - raise err # Re-raise the specific HTTP error on the last attempt + raise # Re-raise the specific HTTP error on the last attempt except RequestException as err: # Catch other request-related errors (e.g., ConnectionError, Timeout) @@ -172,7 +175,7 @@ def get_event_details(event_id: str) -> dict: time.sleep(sleep_for) else: print(f"Max retries ({MAX_RETRIES}) reached for request error. Raising the last error.") - raise err # Re-raise the error on the last attempt + raise # Re-raise the error on the last attempt # This part should ideally not be reached if MAX_RETRIES is set up correctly # and errors are always re-raised on the last attempt. diff --git a/src/django_project/web/utilities/scrapers/meetup.py b/src/django_project/web/utilities/scrapers/meetup.py index a6aebf8..4bcab7e 100644 --- a/src/django_project/web/utilities/scrapers/meetup.py +++ b/src/django_project/web/utilities/scrapers/meetup.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import html import re from datetime import datetime, timedelta, timezone @@ -5,6 +7,7 @@ from bs4 import BeautifulSoup, Tag from bs4.element import NavigableString, PageElement + from web.utilities.html_utils import fetch_content, fetch_content_with_playwright @@ -42,15 +45,14 @@ def get_end_datetime(datetime_string: str, time_string: str) -> datetime | None: elif period == "AM" and hour == 12: hour = 0 - time_obj = datetime.min.replace(hour=hour, minute=minute) + time_obj = datetime.min.replace(tzinfo=tz, hour=hour, minute=minute) # Combine date and time into a new datetime object - combined_datetime: datetime = datetime.combine(datetime.strptime(date_part, "%Y-%m-%d").date(), time_obj.time()) + combined_datetime = datetime.fromisoformat(f"{date_part}T{time_obj.strftime('%H:%M:%S')}{timezone_offset}") # Apply the extracted timezone to the combined datetime - combined_datetime_with_tz: datetime = combined_datetime.replace(tzinfo=tz) - return combined_datetime_with_tz - except Exception as err: + return combined_datetime + except (TypeError, ValueError) as err: print(err) return None @@ -80,70 +82,65 @@ def get_event_information(url: str) -> dict: description_div: PageElement | Tag | NavigableString | None = soup.find( "div", class_="w-full break-words transition-all duration-300 line-clamp-[15]" ) - if description_div: - if isinstance(description_div, Tag): # Type check for Tag - event_info["description"] = "".join(str(child) for child in description_div.children) + if description_div and isinstance(description_div, Tag): # Type check for Tag + event_info["description"] = "".join(str(child) for child in description_div.children) time_element: PageElement | Tag | NavigableString | None = soup.find("time", class_="block") - if time_element: - if isinstance(time_element, Tag): # Check if time_element is a Tag - start_time_string: Any = time_element.get("datetime", None) - time_text: str = time_element.get_text(separator=" ").strip() - - if start_time_string: - if isinstance(start_time_string, str): # Check if start_time_string is a str - start_dt = datetime.fromisoformat(start_time_string) - event_info["start_datetime"] = start_dt - - # Parse duration from the time text which shows times in UTC - # Format: "Friday, Feb 13 · 2:00 AM to 3:00 AM UTC" - # We calculate the duration and add it to start_dt to preserve timezone - if " to " in time_text: - time_parts = time_text.split(" to ") - if len(time_parts) == 2: - # Extract start time from text (in UTC) - start_match = re.search(r"(\d{1,2}):(\d{2})\s*([APap][Mm])", time_parts[0]) - # Extract end time from text (in UTC) - end_match = re.search(r"(\d{1,2}):(\d{2})\s*([APap][Mm])", time_parts[1]) - - if start_match and end_match: - # Parse start time (UTC) - start_hour = int(start_match.group(1)) - start_minute = int(start_match.group(2)) - start_period = start_match.group(3).upper() - if start_period == "PM" and start_hour != 12: - start_hour += 12 - elif start_period == "AM" and start_hour == 12: - start_hour = 0 - - # Parse end time (UTC) - end_hour = int(end_match.group(1)) - end_minute = int(end_match.group(2)) - end_period = end_match.group(3).upper() - if end_period == "PM" and end_hour != 12: - end_hour += 12 - elif end_period == "AM" and end_hour == 12: - end_hour = 0 - - # Calculate duration in minutes - start_minutes = start_hour * 60 + start_minute - end_minutes = end_hour * 60 + end_minute - - # Handle overnight events - if end_minutes <= start_minutes: - end_minutes += 24 * 60 - - duration_minutes = end_minutes - start_minutes - - # Add duration to start_dt to get end_dt in the same timezone - end_dt = start_dt + timedelta(minutes=duration_minutes) - event_info["end_datetime"] = end_dt - else: - event_info["end_datetime"] = None - else: - event_info["end_datetime"] = None + if time_element and isinstance(time_element, Tag): # Check if time_element is a Tag + start_time_string: Any = time_element.get("datetime", None) + time_text: str = time_element.get_text(separator=" ").strip() + + if start_time_string and isinstance(start_time_string, str): + start_dt = datetime.fromisoformat(start_time_string) + event_info["start_datetime"] = start_dt + + # Parse duration from the time text which shows times in UTC + # Format: "Friday, Feb 13 · 2:00 AM to 3:00 AM UTC" + # We calculate the duration and add it to start_dt to preserve timezone + if " to " in time_text: + time_parts = time_text.split(" to ") + if len(time_parts) == 2: + # Extract start time from text (in UTC) + start_match = re.search(r"(\d{1,2}):(\d{2})\s*([APap][Mm])", time_parts[0]) + # Extract end time from text (in UTC) + end_match = re.search(r"(\d{1,2}):(\d{2})\s*([APap][Mm])", time_parts[1]) + + if start_match and end_match: + # Parse start time (UTC) + start_hour = int(start_match.group(1)) + start_minute = int(start_match.group(2)) + start_period = start_match.group(3).upper() + if start_period == "PM" and start_hour != 12: + start_hour += 12 + elif start_period == "AM" and start_hour == 12: + start_hour = 0 + + # Parse end time (UTC) + end_hour = int(end_match.group(1)) + end_minute = int(end_match.group(2)) + end_period = end_match.group(3).upper() + if end_period == "PM" and end_hour != 12: + end_hour += 12 + elif end_period == "AM" and end_hour == 12: + end_hour = 0 + + # Calculate duration in minutes + start_minutes = start_hour * 60 + start_minute + end_minutes = end_hour * 60 + end_minute + + # Handle overnight events + if end_minutes <= start_minutes: + end_minutes += 24 * 60 + + duration_minutes = end_minutes - start_minutes + + # Add duration to start_dt to get end_dt in the same timezone + end_dt = start_dt + timedelta(minutes=duration_minutes) + event_info["end_datetime"] = end_dt else: event_info["end_datetime"] = None + else: + event_info["end_datetime"] = None location_name: str | Any = None match = re.search(r'"__typename":"Venue","id":"\d+","name":"([^"]+)"', page_content) diff --git a/src/django_project/web/views.py b/src/django_project/web/views.py index 0ea03b2..8c38667 100644 --- a/src/django_project/web/views.py +++ b/src/django_project/web/views.py @@ -7,6 +7,7 @@ HtmxOptionView, ModelDetailBootstrapModalView, ) + from web.models import Event, TechGroup