From fb8f2d10984bcfa2585dc272b4c85d285b722792 Mon Sep 17 00:00:00 2001 From: Michael Manfre Date: Mon, 28 Nov 2022 23:41:36 -0500 Subject: [PATCH] Hashtags --- activities/admin.py | 25 ++- activities/migrations/0002_hashtag.py | 51 +++++ activities/models/__init__.py | 1 + activities/models/hashtag.py | 187 ++++++++++++++++++ activities/models/post.py | 96 ++++++--- activities/templatetags/activity_tags.py | 13 ++ activities/views/admin/__init__.py | 0 activities/views/explore.py | 26 +++ activities/views/search.py | 43 +++- activities/views/timelines.py | 45 ++++- core/models/config.py | 3 + docs/features.rst | 2 +- static/css/style.css | 36 ++++ takahe/urls.py | 23 ++- templates/activities/_hashtag.html | 11 ++ templates/activities/_menu.html | 13 +- templates/activities/explore_tag.html | 16 ++ templates/activities/search.html | 8 + templates/activities/tag.html | 16 ++ templates/admin/hashtag_create.html | 26 +++ templates/admin/hashtag_delete.html | 17 ++ templates/admin/hashtag_edit.html | 46 +++++ templates/admin/hashtags.html | 40 ++++ templates/settings/_menu.html | 3 + tests/activities/models/test_hashtag.py | 41 ++++ .../templatetags/test_activity_tags.py | 12 +- users/views/admin/__init__.py | 6 + users/views/admin/hashtags.py | 126 ++++++++++++ users/views/admin/settings.py | 10 +- 29 files changed, 897 insertions(+), 45 deletions(-) create mode 100644 activities/migrations/0002_hashtag.py create mode 100644 activities/models/hashtag.py create mode 100644 activities/views/admin/__init__.py create mode 100644 activities/views/explore.py create mode 100644 templates/activities/_hashtag.html create mode 100644 templates/activities/explore_tag.html create mode 100644 templates/activities/tag.html create mode 100644 templates/admin/hashtag_create.html create mode 100644 templates/admin/hashtag_delete.html create mode 100644 templates/admin/hashtag_edit.html create mode 100644 templates/admin/hashtags.html create mode 100644 tests/activities/models/test_hashtag.py create mode 100644 users/views/admin/hashtags.py diff --git a/activities/admin.py b/activities/admin.py index 8e29d22..c4875ca 100644 --- a/activities/admin.py +++ b/activities/admin.py @@ -1,7 +1,9 @@ +from asgiref.sync import async_to_sync from django.contrib import admin from activities.models import ( FanOut, + Hashtag, Post, PostAttachment, PostInteraction, @@ -9,6 +11,20 @@ from activities.models import ( ) +@admin.register(Hashtag) +class HashtagAdmin(admin.ModelAdmin): + list_display = ["hashtag", "name_override", "state", "stats_updated", "created"] + + readonly_fields = ["created", "updated", "stats_updated"] + + actions = ["force_execution"] + + @admin.action(description="Force Execution") + def force_execution(self, request, queryset): + for instance in queryset: + instance.transition_perform("outdated") + + class PostAttachmentInline(admin.StackedInline): model = PostAttachment extra = 0 @@ -18,7 +34,7 @@ class PostAttachmentInline(admin.StackedInline): class PostAdmin(admin.ModelAdmin): list_display = ["id", "state", "author", "created"] raw_id_fields = ["to", "mentions", "author"] - actions = ["force_fetch"] + actions = ["force_fetch", "reparse_hashtags"] search_fields = ["content"] inlines = [PostAttachmentInline] readonly_fields = ["created", "updated", "object_json"] @@ -28,6 +44,13 @@ class PostAdmin(admin.ModelAdmin): for instance in queryset: instance.debug_fetch() + @admin.action(description="Reprocess content for hashtags") + def reparse_hashtags(self, request, queryset): + for instance in queryset: + instance.hashtags = Hashtag.hashtags_from_content(instance.content) or None + instance.save() + async_to_sync(instance.ensure_hashtags)() + @admin.display(description="ActivityPub JSON") def object_json(self, instance): return instance.to_ap() diff --git a/activities/migrations/0002_hashtag.py b/activities/migrations/0002_hashtag.py new file mode 100644 index 0000000..468bd95 --- /dev/null +++ b/activities/migrations/0002_hashtag.py @@ -0,0 +1,51 @@ +# Generated by Django 4.1.3 on 2022-11-27 20:16 + +from django.db import migrations, models + +import activities.models.hashtag +import stator.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("activities", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="Hashtag", + fields=[ + ("state_ready", models.BooleanField(default=True)), + ("state_changed", models.DateTimeField(auto_now_add=True)), + ("state_attempted", models.DateTimeField(blank=True, null=True)), + ("state_locked_until", models.DateTimeField(blank=True, null=True)), + ( + "hashtag", + models.SlugField(max_length=100, primary_key=True, serialize=False), + ), + ( + "name_override", + models.CharField(blank=True, max_length=100, null=True), + ), + ("public", models.BooleanField(null=True)), + ( + "state", + stator.models.StateField( + choices=[("outdated", "outdated"), ("updated", "updated")], + default="outdated", + graph=activities.models.hashtag.HashtagStates, + max_length=100, + ), + ), + ("stats", models.JSONField(blank=True, null=True)), + ("stats_updated", models.DateTimeField(blank=True, null=True)), + ("aliases", models.JSONField(blank=True, null=True)), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/activities/models/__init__.py b/activities/models/__init__.py index 1ae3f4c..aa34c0f 100644 --- a/activities/models/__init__.py +++ b/activities/models/__init__.py @@ -1,4 +1,5 @@ from .fan_out import FanOut, FanOutStates # noqa +from .hashtag import Hashtag, HashtagStates # noqa from .post import Post, PostStates # noqa from .post_attachment import PostAttachment, PostAttachmentStates # noqa from .post_interaction import PostInteraction, PostInteractionStates # noqa diff --git a/activities/models/hashtag.py b/activities/models/hashtag.py new file mode 100644 index 0000000..a5754f7 --- /dev/null +++ b/activities/models/hashtag.py @@ -0,0 +1,187 @@ +import re +from datetime import date, timedelta +from typing import Dict, List + +import urlman +from asgiref.sync import sync_to_async +from django.db import models +from django.utils import timezone +from django.utils.safestring import mark_safe + +from core.models import Config +from stator.models import State, StateField, StateGraph, StatorModel + + +class HashtagStates(StateGraph): + outdated = State(try_interval=300, force_initial=True) + updated = State(try_interval=3600, attempt_immediately=False) + + outdated.transitions_to(updated) + updated.transitions_to(outdated) + + @classmethod + async def handle_outdated(cls, instance: "Hashtag"): + """ + Computes the stats and other things for a Hashtag + """ + from .post import Post + + posts_query = Post.objects.local_public().tagged_with(instance) + total = await posts_query.acount() + + today = timezone.now().date() + # TODO: single query + total_today = await posts_query.filter( + created__gte=today, + created__lte=today + timedelta(days=1), + ).acount() + total_month = await posts_query.filter( + created__year=today.year, + created__month=today.month, + ).acount() + total_year = await posts_query.filter( + created__year=today.year, + ).acount() + if total: + if not instance.stats: + instance.stats = {} + instance.stats.update( + { + "total": total, + today.isoformat(): total_today, + today.strftime("%Y-%m"): total_month, + today.strftime("%Y"): total_year, + } + ) + instance.stats_updated = timezone.now() + await sync_to_async(instance.save)() + + return cls.updated + + @classmethod + async def handle_updated(cls, instance: "Hashtag"): + if instance.state_age > Config.system.hashtag_stats_max_age: + return cls.outdated + + +class HashtagQuerySet(models.QuerySet): + def public(self): + public_q = models.Q(public=True) + if Config.system.hashtag_unreviewed_are_public: + public_q |= models.Q(public__isnull=True) + return self.filter(public_q) + + def hashtag_or_alias(self, hashtag: str): + return self.filter( + models.Q(hashtag=hashtag) | models.Q(aliases__contains=hashtag) + ) + + +class HashtagManager(models.Manager): + def get_queryset(self): + return HashtagQuerySet(self.model, using=self._db) + + def public(self): + return self.get_queryset().public() + + def hashtag_or_alias(self, hashtag: str): + return self.get_queryset().hashtag_or_alias(hashtag) + + +class Hashtag(StatorModel): + + # Normalized hashtag without the '#' + hashtag = models.SlugField(primary_key=True, max_length=100) + + # Friendly display override + name_override = models.CharField(max_length=100, null=True, blank=True) + + # Should this be shown in the public UI? + public = models.BooleanField(null=True) + + # State of this Hashtag + state = StateField(HashtagStates) + + # Metrics for this Hashtag + stats = models.JSONField(null=True, blank=True) + # Timestamp of last time the stats were updated + stats_updated = models.DateTimeField(null=True, blank=True) + + # List of other hashtags that are considered similar + aliases = models.JSONField(null=True, blank=True) + + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + objects = HashtagManager() + + class urls(urlman.Urls): + root = "/admin/hashtags/" + create = "/admin/hashtags/create/" + edit = "/admin/hashtags/{self.hashtag}/" + delete = "{edit}delete/" + timeline = "/tags/{self.hashtag}/" + + hashtag_regex = re.compile(r"((?:\B#)([a-zA-Z0-9(_)]{1,}\b))") + + def save(self, *args, **kwargs): + self.hashtag = self.hashtag.lstrip("#") + if self.name_override: + self.name_override = self.name_override.lstrip("#") + return super().save(*args, **kwargs) + + @property + def display_name(self): + return self.name_override or self.hashtag + + def __str__(self): + return self.display_name + + def usage_months(self, num: int = 12) -> Dict[date, int]: + """ + Return the most recent num months of stats + """ + if not self.stats: + return {} + results = {} + for key, val in self.stats.items(): + parts = key.split("-") + if len(parts) == 2: + year = int(parts[0]) + month = int(parts[1]) + results[date(year, month, 1)] = val + return dict(sorted(results.items(), reverse=True)[:num]) + + def usage_days(self, num: int = 7) -> Dict[date, int]: + """ + Return the most recent num days of stats + """ + if not self.stats: + return {} + results = {} + for key, val in self.stats.items(): + parts = key.split("-") + if len(parts) == 3: + year = int(parts[0]) + month = int(parts[1]) + day = int(parts[2]) + results[date(year, month, day)] = val + return dict(sorted(results.items(), reverse=True)[:num]) + + @classmethod + def hashtags_from_content(cls, content) -> List[str]: + """ + Return a parsed and sanitized of hashtags found in content without + leading '#'. + """ + hashtag_hits = cls.hashtag_regex.findall(content) + hashtags = sorted({tag[1].lower() for tag in hashtag_hits}) + return list(hashtags) + + @classmethod + def linkify_hashtags(cls, content) -> str: + def replacer(match): + hashtag = match.group() + return f'{hashtag}' + + return mark_safe(Hashtag.hashtag_regex.sub(replacer, content)) diff --git a/activities/models/post.py b/activities/models/post.py index f504fcb..b61abd4 100644 --- a/activities/models/post.py +++ b/activities/models/post.py @@ -10,6 +10,7 @@ from django.utils import timezone from django.utils.safestring import mark_safe from activities.models.fan_out import FanOut +from activities.models.hashtag import Hashtag from core.html import sanitize_post, strip_html from core.ld import canonicalise, format_ld_date, get_list, parse_ld_date from stator.models import State, StateField, StateGraph, StatorModel @@ -34,19 +35,24 @@ class PostStates(StateGraph): edited_fanned_out.transitions_to(edited) edited_fanned_out.transitions_to(deleted) + @classmethod + async def targets_fan_out(cls, post: "Post", type_: str) -> None: + # Fan out to each target + for follow in await post.aget_targets(): + await FanOut.objects.acreate( + identity=follow, + type=type_, + subject_post=post, + ) + @classmethod async def handle_new(cls, instance: "Post"): """ Creates all needed fan-out objects for a new Post. """ post = await instance.afetch_full() - # Fan out to each target - for follow in await post.aget_targets(): - await FanOut.objects.acreate( - identity=follow, - type=FanOut.Types.post, - subject_post=post, - ) + await cls.targets_fan_out(post, FanOut.Types.post) + await post.ensure_hashtags() return cls.fanned_out @classmethod @@ -55,13 +61,7 @@ class PostStates(StateGraph): Creates all needed fan-out objects needed to delete a Post. """ post = await instance.afetch_full() - # Fan out to each target - for follow in await post.aget_targets(): - await FanOut.objects.acreate( - identity=follow, - type=FanOut.Types.post_deleted, - subject_post=post, - ) + await cls.targets_fan_out(post, FanOut.Types.post_deleted) return cls.deleted_fanned_out @classmethod @@ -70,16 +70,46 @@ class PostStates(StateGraph): Creates all needed fan-out objects for an edited Post. """ post = await instance.afetch_full() - # Fan out to each target - for follow in await post.aget_targets(): - await FanOut.objects.acreate( - identity=follow, - type=FanOut.Types.post_edited, - subject_post=post, - ) + await cls.targets_fan_out(post, FanOut.Types.post_edited) + await post.ensure_hashtags() return cls.edited_fanned_out +class PostQuerySet(models.QuerySet): + def local_public(self, include_replies: bool = False): + query = self.filter( + visibility__in=[ + Post.Visibilities.public, + Post.Visibilities.local_only, + ], + author__local=True, + ) + if not include_replies: + return query.filter(in_reply_to__isnull=True) + return query + + def tagged_with(self, hashtag: str | Hashtag): + if isinstance(hashtag, str): + tag_q = models.Q(hashtags__contains=hashtag) + else: + tag_q = models.Q(hashtags__contains=hashtag.hashtag) + if hashtag.aliases: + for alias in hashtag.aliases: + tag_q |= models.Q(hashtags__contains=alias) + return self.filter(tag_q) + + +class PostManager(models.Manager): + def get_queryset(self): + return PostQuerySet(self.model, using=self._db) + + def local_public(self, include_replies: bool = False): + return self.get_queryset().local_public(include_replies=include_replies) + + def tagged_with(self, hashtag: str | Hashtag): + return self.get_queryset().tagged_with(hashtag=hashtag) + + class Post(StatorModel): """ A post (status, toot) that is either local or remote. @@ -155,6 +185,8 @@ class Post(StatorModel): created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) + objects = PostManager() + class urls(urlman.Urls): view = "{self.author.urls.view}posts/{self.id}/" object_uri = "{self.author.actor_uri}posts/{self.id}/" @@ -236,7 +268,9 @@ class Post(StatorModel): """ Returns the content formatted for local display """ - return self.linkify_mentions(sanitize_post(self.content), local=True) + return Hashtag.linkify_hashtags( + self.linkify_mentions(sanitize_post(self.content), local=True) + ) def safe_content_remote(self): """ @@ -252,7 +286,7 @@ class Post(StatorModel): ### Async helpers ### - async def afetch_full(self): + async def afetch_full(self) -> "Post": """ Returns a version of the object with all relations pre-loaded """ @@ -281,6 +315,8 @@ class Post(StatorModel): # Maintain local-only for replies if reply_to.visibility == reply_to.Visibilities.local_only: visibility = reply_to.Visibilities.local_only + # Find hashtags in this post + hashtags = Hashtag.hashtags_from_content(content) or None # Strip all HTML and apply linebreaks filter content = linebreaks_filter(strip_html(content)) # Make the Post object @@ -291,6 +327,7 @@ class Post(StatorModel): sensitive=bool(summary), local=True, visibility=visibility, + hashtags=hashtags, in_reply_to=reply_to.object_uri if reply_to else None, ) post.object_uri = post.urls.object_uri @@ -312,6 +349,7 @@ class Post(StatorModel): self.sensitive = bool(summary) self.visibility = visibility self.edited = timezone.now() + self.hashtags = Hashtag.hashtags_from_content(content) or None self.mentions.set(self.mentions_from_content(content, self.author)) self.save() @@ -334,6 +372,18 @@ class Post(StatorModel): mentions.add(identity) return mentions + async def ensure_hashtags(self) -> None: + """ + Ensure any of the already parsed hashtags from this Post + have a corresponding Hashtag record. + """ + # Ensure hashtags + if self.hashtags: + for hashtag in self.hashtags: + await Hashtag.objects.aget_or_create( + hashtag=hashtag, + ) + ### ActivityPub (outbound) ### def to_ap(self) -> Dict: diff --git a/activities/templatetags/activity_tags.py b/activities/templatetags/activity_tags.py index 571e2d6..fb822f6 100644 --- a/activities/templatetags/activity_tags.py +++ b/activities/templatetags/activity_tags.py @@ -3,6 +3,8 @@ import datetime from django import template from django.utils import timezone +from activities.models import Hashtag + register = template.Library() @@ -31,3 +33,14 @@ def timedeltashort(value: datetime.datetime): years = max(days // 365.25, 1) text = f"{years:0n}y" return text + + +@register.filter +def linkify_hashtags(value: str): + """ + Convert hashtags in content in to /tags// links. + """ + if not value: + return "" + + return Hashtag.linkify_hashtags(value) diff --git a/activities/views/admin/__init__.py b/activities/views/admin/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/activities/views/explore.py b/activities/views/explore.py new file mode 100644 index 0000000..ddb1e6c --- /dev/null +++ b/activities/views/explore.py @@ -0,0 +1,26 @@ +from django.views.generic import ListView + +from activities.models import Hashtag + + +class ExploreTag(ListView): + + template_name = "activities/explore_tag.html" + extra_context = { + "current_page": "explore", + "allows_refresh": True, + } + paginate_by = 20 + + def get_queryset(self): + return ( + Hashtag.objects.public() + .filter( + stats__total__gt=0, + ) + .order_by("-stats__total") + )[:20] + + +class Explore(ExploreTag): + pass diff --git a/activities/views/search.py b/activities/views/search.py index b175052..4719f64 100644 --- a/activities/views/search.py +++ b/activities/views/search.py @@ -1,6 +1,9 @@ +from typing import Set + from django import forms from django.views.generic import FormView +from activities.models import Hashtag from users.models import Domain, Identity @@ -9,13 +12,13 @@ class Search(FormView): template_name = "activities/search.html" class form_class(forms.Form): - query = forms.CharField(help_text="Search for a user by @username@domain") - - def form_valid(self, form): - query = form.cleaned_data["query"].lstrip("@").lower() - results = {"identities": set()} - # Search identities + query = forms.CharField( + help_text="Search for a user by @username@domain or hashtag by #tagname" + ) + def search_identities(self, query: str): + query = query.lstrip("@") + results: Set[Identity] = set() if "@" in query: username, domain = query.split("@", 1) @@ -35,13 +38,35 @@ class Search(FormView): ) identity = None if identity: - results["identities"].add(identity) + results.add(identity) else: for identity in Identity.objects.filter(username=query)[:20]: - results["identities"].add(identity) + results.add(identity) for identity in Identity.objects.filter(username__startswith=query)[:20]: - results["identities"].add(identity) + results.add(identity) + return results + + def search_hashtags(self, query: str): + results: Set[Hashtag] = set() + + if "@" in query: + return results + + query = query.lstrip("#") + for hashtag in Hashtag.objects.public().hashtag_or_alias(query)[:10]: + results.add(hashtag) + for hashtag in Hashtag.objects.public().filter(hashtag__startswith=query)[:10]: + results.add(hashtag) + return results + + def form_valid(self, form): + query = form.cleaned_data["query"].lower() + results = { + "identities": self.search_identities(query), + "hashtags": self.search_hashtags(query), + } + # Render results context = self.get_context_data(form=form) context["results"] = results diff --git a/activities/views/timelines.py b/activities/views/timelines.py index 4f2a515..ffe329c 100644 --- a/activities/views/timelines.py +++ b/activities/views/timelines.py @@ -1,10 +1,10 @@ from django import forms -from django.shortcuts import redirect +from django.shortcuts import get_object_or_404, redirect from django.template.defaultfilters import linebreaks_filter from django.utils.decorators import method_decorator from django.views.generic import FormView, ListView -from activities.models import Post, PostInteraction, TimelineEvent +from activities.models import Hashtag, Post, PostInteraction, TimelineEvent from core.models import Config from users.decorators import identity_required @@ -61,6 +61,41 @@ class Home(FormView): return redirect(".") +class Tag(ListView): + + template_name = "activities/tag.html" + extra_context = { + "current_page": "tag", + "allows_refresh": True, + } + paginate_by = 50 + + def get(self, request, hashtag, *args, **kwargs): + tag = hashtag.lower().lstrip("#") + if hashtag != tag: + # SEO sanitize + return redirect(f"/tags/{tag}/", permanent=True) + self.hashtag = get_object_or_404(Hashtag.objects.public(), hashtag=tag) + return super().get(request, *args, **kwargs) + + def get_queryset(self): + return ( + Post.objects.local_public() + .tagged_with(self.hashtag) + .select_related("author") + .prefetch_related("attachments") + .order_by("-created")[:50] + ) + + def get_context_data(self): + context = super().get_context_data() + context["hashtag"] = self.hashtag + context["interactions"] = PostInteraction.get_post_interactions( + context["page_obj"], self.request.identity + ) + return context + + class Local(ListView): template_name = "activities/local.html" @@ -72,11 +107,7 @@ class Local(ListView): def get_queryset(self): return ( - Post.objects.filter( - visibility=Post.Visibilities.public, - author__local=True, - in_reply_to__isnull=True, - ) + Post.objects.local_public() .select_related("author") .prefetch_related("attachments") .order_by("-created")[:50] diff --git a/core/models/config.py b/core/models/config.py index dab0059..dd7da9f 100644 --- a/core/models/config.py +++ b/core/models/config.py @@ -215,6 +215,9 @@ class Config(models.Model): identity_max_age: int = 24 * 60 * 60 inbox_message_purge_after: int = 24 * 60 * 60 + hashtag_unreviewed_are_public: bool = True + hashtag_stats_max_age: int = 60 * 60 + restricted_usernames: str = "admin\nadmins\nadministrator\nadministrators\nsystem\nroot\nannounce\nannouncement\nannouncements" class UserOptions(pydantic.BaseModel): diff --git a/docs/features.rst b/docs/features.rst index 35ec53c..27ea102 100644 --- a/docs/features.rst +++ b/docs/features.rst @@ -22,6 +22,7 @@ Currently, it supports: * Server defederation (blocking) * Signup flow * Password reset flow +* Hashtag trending system with moderation Features planned for releases up to 1.0: @@ -40,7 +41,6 @@ Features that may make it into 1.0, or might be further out: * Creating polls on posts, and handling received polls * Filter system for Home timeline -* Hashtag trending system with moderation * Mastodon-compatible account migration target/source * Relay support diff --git a/static/css/style.css b/static/css/style.css index 1dbf30e..ebbedb0 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -448,6 +448,23 @@ form .field .label-input { flex-grow: 1; } +form .field.stats { + width: 100%; +} +form .field.stats table { + width: 50%; +} + +form .field.stats table tr th { + color: var(--color-text-main); +} + + +form .field.stats table tbody td { + color: var(--color-text-dull); + text-align: center; +} + .right-column form .field { margin: 0; background: none; @@ -704,6 +721,17 @@ table.metadata td.name { font-weight: bold; } +/* Named Timelines */ + +.left-column .timeline-name { + margin: 0 0 10px 0; + color: var(--color-text-main); + font-size: 130%; +} + +.left-column .timeline-name i { + margin-right: 10px +} /* Posts */ @@ -879,6 +907,14 @@ table.metadata td.name { width: 16px; } +.post a.hashtag, .post.mini a.hashtag { + text-decoration: none; +} + +.post a.hashtag:hover, .post.mini a.hashtag:hover { + text-decoration: underline; +} + .boost-banner, .mention-banner, .follow-banner, diff --git a/takahe/urls.py b/takahe/urls.py index 6f5ac79..dc3946f 100644 --- a/takahe/urls.py +++ b/takahe/urls.py @@ -3,7 +3,7 @@ from django.contrib import admin as djadmin from django.urls import path, re_path from django.views.static import serve -from activities.views import posts, search, timelines +from activities.views import explore, posts, search, timelines from core import views as core from stator import views as stator from users.views import activitypub, admin, auth, follows, identity, settings @@ -16,6 +16,9 @@ urlpatterns = [ path("local/", timelines.Local.as_view(), name="local"), path("federated/", timelines.Federated.as_view(), name="federated"), path("search/", search.Search.as_view(), name="search"), + path("tags//", timelines.Tag.as_view(), name="tag"), + path("explore/", explore.Explore.as_view(), name="explore"), + path("explore/tags/", explore.ExploreTag.as_view(), name="explore-tag"), path( "settings/", settings.SettingsRoot.as_view(), @@ -94,6 +97,24 @@ urlpatterns = [ admin.Invites.as_view(), name="admin_invites", ), + path( + "admin/hashtags/", + admin.Hashtags.as_view(), + name="admin_hashtags", + ), + path( + "admin/hashtags/create/", + admin.HashtagCreate.as_view(), + name="admin_hashtags_create", + ), + path( + "admin/hashtags//", + admin.HashtagEdit.as_view(), + ), + path( + "admin/hashtags//delete/", + admin.HashtagDelete.as_view(), + ), # Identity views path("@/", identity.ViewIdentity.as_view()), path("@/inbox/", activitypub.Inbox.as_view()), diff --git a/templates/activities/_hashtag.html b/templates/activities/_hashtag.html new file mode 100644 index 0000000..19233e5 --- /dev/null +++ b/templates/activities/_hashtag.html @@ -0,0 +1,11 @@ + + + + {{ hashtag.display_name }} + + {% if not hide_stats %} + + Post count: {{ hashtag.stats.total }} + + {% endif %} + diff --git a/templates/activities/_menu.html b/templates/activities/_menu.html index 1ebe940..58295a9 100644 --- a/templates/activities/_menu.html +++ b/templates/activities/_menu.html @@ -6,6 +6,9 @@ Notifications + + Explore + Local @@ -19,13 +22,21 @@ Search + {% if current_page == "tag" %} + + {{ hashtag.display_name }} + + {% endif %} Settings - {% else %} + {% else %} Local Posts + + Explore +

{% if config.signup_allowed %} diff --git a/templates/activities/explore_tag.html b/templates/activities/explore_tag.html new file mode 100644 index 0000000..b2fd79d --- /dev/null +++ b/templates/activities/explore_tag.html @@ -0,0 +1,16 @@ +{% extends "base.html" %} + +{% block title %}#{{ hashtag.display_name }} Timeline{% endblock %} + +{% block content %} +
Explore Trending Tags
+ +
+ {% for hashtag in page_obj %} + {% include "activities/_hashtag.html" %} + {% empty %} + No tags are trending yet. + {% endfor %} + +
+{% endblock %} diff --git a/templates/activities/search.html b/templates/activities/search.html index 3cff2a2..5137740 100644 --- a/templates/activities/search.html +++ b/templates/activities/search.html @@ -18,4 +18,12 @@ {% include "activities/_identity.html" %} {% endfor %} {% endif %} + {% if results.hashtags %} +

Hashtags

+
+ {% for hashtag in results.hashtags %} + {% include "activities/_hashtag.html" with hide_stats=True %} + {% endfor %} +
+ {% endif %} {% endblock %} diff --git a/templates/activities/tag.html b/templates/activities/tag.html new file mode 100644 index 0000000..a319b6a --- /dev/null +++ b/templates/activities/tag.html @@ -0,0 +1,16 @@ +{% extends "base.html" %} + +{% block title %}#{{ hashtag.display_name }} Timeline{% endblock %} + +{% block content %} +
{{ hashtag.display_name }}
+ {% for post in page_obj %} + {% include "activities/_post.html" %} + {% empty %} + No posts yet. + {% endfor %} + + {% if page_obj.has_next %} +
Next Page
+ {% endif %} +{% endblock %} diff --git a/templates/admin/hashtag_create.html b/templates/admin/hashtag_create.html new file mode 100644 index 0000000..2d31cf7 --- /dev/null +++ b/templates/admin/hashtag_create.html @@ -0,0 +1,26 @@ +{% extends "settings/base.html" %} + +{% block title %}Add hashtag - Admin{% endblock %} + +{% block content %} +
+

Add A hashtag

+

+ Use this form to add a hashtag. +

+ {% csrf_token %} +
+ hashtag Details + {% include "forms/_field.html" with field=form.hashtag %} + {% include "forms/_field.html" with field=form.name_override %} +
+
+ Access Control + {% include "forms/_field.html" with field=form.public %} +
+
+ Back + +
+
+{% endblock %} diff --git a/templates/admin/hashtag_delete.html b/templates/admin/hashtag_delete.html new file mode 100644 index 0000000..9aca4e7 --- /dev/null +++ b/templates/admin/hashtag_delete.html @@ -0,0 +1,17 @@ +{% extends "settings/base.html" %} + +{% block title %}Delete {{ hashtag.hashtag }} - Admin{% endblock %} + +{% block content %} +
+ {% csrf_token %} + +

Deleting {{ hashtag.hashtag }}

+ +

Please confirm deletion of this hashtag.

+
+ Cancel + +
+ +{% endblock %} diff --git a/templates/admin/hashtag_edit.html b/templates/admin/hashtag_edit.html new file mode 100644 index 0000000..b023dfa --- /dev/null +++ b/templates/admin/hashtag_edit.html @@ -0,0 +1,46 @@ +{% extends "settings/base.html" %} + +{% block subtitle %}{{ hashtag.hashtag }}{% endblock %} + +{% block content %} + + {% csrf_token %} +
+ hashtag Details + {% include "forms/_field.html" with field=form.hashtag %} + {% include "forms/_field.html" with field=form.name_override %} +
+
+ Access Control + {% include "forms/_field.html" with field=form.public %} +
+
+ Stats +
+ {% for stat_month, stat_value in hashtag.usage_months.items|slice:":5" %} + {% if forloop.first %} + + + + + + {% endif %} + + + + + {% if forloop.last %} +
MonthUsage
{{ stat_month|date:"M Y" }}{{ stat_value }}
+ {% endif %} + {% empty %} +

Hashtag is either not used or stats have not been computed yet.

+ {% endfor %} +
+
+
+ Back + Delete + +
+
+{% endblock %} diff --git a/templates/admin/hashtags.html b/templates/admin/hashtags.html new file mode 100644 index 0000000..4273ac2 --- /dev/null +++ b/templates/admin/hashtags.html @@ -0,0 +1,40 @@ +{% extends "settings/base.html" %} + +{% block subtitle %}Hashtags{% endblock %} + +{% block content %} +
+ {% for hashtag in hashtags %} + + + + {{ hashtag.display_name }} + + {% if hashtag.public %}Public{% elif hashtag.public is None %}Unreviewed{% else %}Private{% endif %} + + + {% if hashtag.stats %} + + Total: + {{ hashtag.stats.total }} + + {% endif %} + {% if hashtag.aliases %} + + + Aliases: + {% for alias in hashtag.aliases %} + {{ alias }}{% if not forloop.last %}, {% endif %} + {% endfor %} + + {% endif %} + + + {% empty %} +

You have no hashtags set up.

+ {% endfor %} + + Add a hashtag + +
+{% endblock %} diff --git a/templates/settings/_menu.html b/templates/settings/_menu.html index 531febb..8aede68 100644 --- a/templates/settings/_menu.html +++ b/templates/settings/_menu.html @@ -36,6 +36,9 @@ Invites + + Hashtags + Django Admin diff --git a/tests/activities/models/test_hashtag.py b/tests/activities/models/test_hashtag.py new file mode 100644 index 0000000..32742d6 --- /dev/null +++ b/tests/activities/models/test_hashtag.py @@ -0,0 +1,41 @@ +from activities.models import Hashtag + + +def test_hashtag_from_content(): + assert Hashtag.hashtags_from_content("#hashtag") == ["hashtag"] + assert Hashtag.hashtags_from_content("a#hashtag") == [] + assert Hashtag.hashtags_from_content("Text #with #hashtag in it") == [ + "hashtag", + "with", + ] + assert Hashtag.hashtags_from_content("#hashtag.") == ["hashtag"] + assert Hashtag.hashtags_from_content("More text\n#one # two ##three #hashtag;") == [ + "hashtag", + "one", + "three", + ] + + +def test_linkify_hashtag(): + linkify = Hashtag.linkify_hashtags + + assert linkify("# hashtag") == "# hashtag" + assert ( + linkify('Text') + == 'Text' + ) + assert ( + linkify("#HashTag") == '#HashTag' + ) + assert ( + linkify( + """A longer text #bigContent +with #tags, linebreaks, and +maybe a few links +#allTheTags #AllTheTags #ALLTHETAGS""" + ) + == """A longer text #bigContent +with #tags, linebreaks, and +maybe a few links +#allTheTags #AllTheTags #ALLTHETAGS""" + ) diff --git a/tests/activities/templatetags/test_activity_tags.py b/tests/activities/templatetags/test_activity_tags.py index 987c008..85d8cdf 100644 --- a/tests/activities/templatetags/test_activity_tags.py +++ b/tests/activities/templatetags/test_activity_tags.py @@ -2,7 +2,7 @@ from datetime import timedelta from django.utils import timezone -from activities.templatetags.activity_tags import timedeltashort +from activities.templatetags.activity_tags import linkify_hashtags, timedeltashort def test_timedeltashort_regress(): @@ -19,3 +19,13 @@ def test_timedeltashort_regress(): assert timedeltashort(value - timedelta(days=364)) == "364d" assert timedeltashort(value - timedelta(days=365)) == "1y" assert timedeltashort(value - timedelta(days=366)) == "1y" + + +def test_linkify_hashtags_regres(): + assert linkify_hashtags(None) == "" + assert linkify_hashtags("") == "" + + assert ( + linkify_hashtags("#Takahe") + == '#Takahe' + ) diff --git a/users/views/admin/__init__.py b/users/views/admin/__init__.py index d1a4db1..101ca30 100644 --- a/users/views/admin/__init__.py +++ b/users/views/admin/__init__.py @@ -11,6 +11,12 @@ from users.views.admin.domains import ( # noqa Domains, ) from users.views.admin.federation import FederationEdit, FederationRoot # noqa +from users.views.admin.hashtags import ( # noqa + HashtagCreate, + HashtagDelete, + HashtagEdit, + Hashtags, +) from users.views.admin.settings import BasicSettings # noqa diff --git a/users/views/admin/hashtags.py b/users/views/admin/hashtags.py new file mode 100644 index 0000000..90f7a84 --- /dev/null +++ b/users/views/admin/hashtags.py @@ -0,0 +1,126 @@ +from django import forms +from django.shortcuts import get_object_or_404, redirect +from django.utils.decorators import method_decorator +from django.views.generic import FormView, TemplateView + +from activities.models import Hashtag, HashtagStates +from users.decorators import admin_required + + +@method_decorator(admin_required, name="dispatch") +class Hashtags(TemplateView): + + template_name = "admin/hashtags.html" + + def get_context_data(self): + return { + "hashtags": Hashtag.objects.filter().order_by("hashtag"), + "section": "hashtag", + } + + +@method_decorator(admin_required, name="dispatch") +class HashtagCreate(FormView): + + template_name = "admin/hashtag_create.html" + extra_context = {"section": "hashtags"} + + class form_class(forms.Form): + hashtag = forms.SlugField( + help_text="The hashtag without the '#'", + ) + name_override = forms.CharField( + help_text="Optional - a more human readable hashtag.", + required=False, + ) + public = forms.NullBooleanField( + help_text="Should this hashtag appear in the UI", + widget=forms.Select( + choices=[(None, "Unreviewed"), (True, "Public"), (False, "Private")] + ), + required=False, + ) + + def clean_hashtag(self): + hashtag = self.cleaned_data["hashtag"].lstrip("#").lower() + if not Hashtag.hashtag_regex.match("#" + hashtag): + raise forms.ValidationError("This does not look like a hashtag name") + if Hashtag.objects.filter(hashtag=hashtag): + raise forms.ValidationError("This hashtag name is already in use") + return hashtag + + def clean_name_override(self): + name_override = self.cleaned_data["name_override"] + if not name_override: + return None + if self.cleaned_data["hashtag"] != name_override.lower(): + raise forms.ValidationError( + "Name override doesn't match hashtag. Only case changes are allowed." + ) + return self.cleaned_data["name_override"] + + def form_valid(self, form): + Hashtag.objects.create( + hashtag=form.cleaned_data["hashtag"], + name_override=form.cleaned_data["name_override"] or None, + public=form.cleaned_data["public"], + ) + return redirect(Hashtag.urls.root) + + +@method_decorator(admin_required, name="dispatch") +class HashtagEdit(FormView): + + template_name = "admin/hashtag_edit.html" + extra_context = {"section": "hashtags"} + + class form_class(HashtagCreate.form_class): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["hashtag"].disabled = True + + def clean_hashtag(self): + return self.cleaned_data["hashtag"] + + def dispatch(self, request, hashtag): + self.hashtag = get_object_or_404(Hashtag.objects, hashtag=hashtag) + return super().dispatch(request) + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context["hashtag"] = self.hashtag + return context + + def form_valid(self, form): + self.hashtag.public = form.cleaned_data["public"] + self.hashtag.name_override = form.cleaned_data["name_override"] + self.hashtag.save() + Hashtag.transition_perform(self.hashtag, HashtagStates.outdated) + return redirect(Hashtag.urls.root) + + def get_initial(self): + return { + "hashtag": self.hashtag.hashtag, + "name_override": self.hashtag.name_override, + "public": self.hashtag.public, + } + + +@method_decorator(admin_required, name="dispatch") +class HashtagDelete(TemplateView): + + template_name = "admin/hashtag_delete.html" + + def dispatch(self, request, hashtag): + self.hashtag = get_object_or_404(Hashtag.objects, hashtag=hashtag) + return super().dispatch(request) + + def get_context_data(self): + return { + "hashtag": self.hashtag, + "section": "hashtags", + } + + def post(self, request): + self.hashtag.delete() + return redirect("admin_hashtags") diff --git a/users/views/admin/settings.py b/users/views/admin/settings.py index 44a338f..a9ec78b 100644 --- a/users/views/admin/settings.py +++ b/users/views/admin/settings.py @@ -80,6 +80,10 @@ class BasicSettings(AdminSettingsPage): "help_text": "Usernames that only admins can register for identities. One per line.", "display": "textarea", }, + "hashtag_unreviewed_are_public": { + "title": "Unreviewed Hashtags Are Public", + "help_text": "Public Hashtags may appear in Trending and have a Tags timeline", + }, } layout = { @@ -91,7 +95,11 @@ class BasicSettings(AdminSettingsPage): "highlight_color", ], "Signups": ["signup_allowed", "signup_invite_only", "signup_text"], - "Posts": ["post_length", "content_warning_text"], + "Posts": [ + "post_length", + "content_warning_text", + "hashtag_unreviewed_are_public", + ], "Identities": [ "identity_max_per_user", "identity_min_length",