Add Domain.state with nodeinfo fetching (#347)

stator/models.py

@@ -102,6 +102,9 @@ class StatorModel(models.Model):
 
     class Meta:
         abstract = True
+        # Need this empty indexes to ensure child Models have a Meta.indexes
+        # that will look to add indexes (that we inject with class_prepared)
+        indexes: list = []
 
     def __init_subclass__(cls) -> None:
         if cls is not StatorModel:

tests/conftest.py

@@ -95,13 +95,17 @@ def user() -> User:
 @pytest.fixture
 @pytest.mark.django_db
 def domain() -> Domain:
-    return Domain.objects.create(domain="example.com", local=True, public=True)
+    return Domain.objects.create(
+        domain="example.com", local=True, public=True, state="updated"
+    )
 
 
 @pytest.fixture
 @pytest.mark.django_db
 def domain2() -> Domain:
-    return Domain.objects.create(domain="example2.com", local=True, public=True)
+    return Domain.objects.create(
+        domain="example2.com", local=True, public=True, state="updated"
+    )
 
 
 @pytest.fixture
@@ -164,7 +168,7 @@ def remote_identity() -> Identity:
     """
     Creates a basic remote test identity with a domain.
     """
-    domain = Domain.objects.create(domain="remote.test", local=False)
+    domain = Domain.objects.create(domain="remote.test", local=False, state="updated")
     return Identity.objects.create(
         actor_uri="https://remote.test/test-actor/",
         inbox_uri="https://remote.test/@test/inbox/",

users/admin.py

@@ -1,5 +1,6 @@
 from django.contrib import admin
 from django.db import models
+from django.utils import formats
 from django.utils.translation import gettext_lazy as _
 
 from activities.admin import IdentityLocalFilter
@@ -18,9 +19,60 @@ from users.models import (
 
 @admin.register(Domain)
 class DomainAdmin(admin.ModelAdmin):
-    list_display = ["domain", "service_domain", "local", "blocked", "public"]
+    list_display = [
+        "domain",
+        "service_domain",
+        "local",
+        "blocked",
+        "software",
+        "user_count",
+        "public",
+    ]
     list_filter = ("local", "blocked")
     search_fields = ("domain", "service_domain")
+    actions = ["force_outdated", "force_updated", "force_connection_issue"]
+
+    @admin.action(description="Force State: outdated")
+    def force_outdated(self, request, queryset):
+        for instance in queryset:
+            instance.transition_perform("outdated")
+
+    @admin.action(description="Force State: updated")
+    def force_updated(self, request, queryset):
+        for instance in queryset:
+            instance.transition_perform("updated")
+
+    @admin.action(description="Force State: connection_issue")
+    def force_connection_issue(self, request, queryset):
+        for instance in queryset:
+            instance.transition_perform("connection_issue")
+
+    @admin.display(description="Software")
+    def software(self, instance):
+        if instance.nodeinfo:
+            software = instance.nodeinfo.get("software", {})
+            name = software.get("name", "unknown")
+            version = software.get("version", "unknown")
+            return f"{name:.10} - {version:.10}"
+
+        return "-"
+
+    @admin.display(description="# Users")
+    def user_count(self, instance):
+        if instance.nodeinfo:
+            usage = instance.nodeinfo.get("usage", {})
+            total = usage.get("users", {}).get("total")
+            if total:
+                try:
+                    return formats.number_format(
+                        "%d" % (int(total)),
+                        0,
+                        use_l10n=True,
+                        force_grouping=True,
+                    )
+                except ValueError:
+                    pass
+        return "-"
 
 
 @admin.register(User)

users/migrations/0010_domain_state.py

@@ -0,0 +1,74 @@
+# Generated by Django 4.1.4 on 2023-01-02 03:54
+
+import django.utils.timezone
+from django.db import migrations, models
+
+import stator.models
+import users.models.domain
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("users", "0009_state_and_post_indexes"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="domain",
+            name="nodeinfo",
+            field=models.JSONField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="domain",
+            name="state",
+            field=stator.models.StateField(
+                choices=[
+                    ("outdated", "outdated"),
+                    ("updated", "updated"),
+                    ("connection_issue", "connection_issue"),
+                    ("purged", "purged"),
+                ],
+                default="outdated",
+                graph=users.models.domain.DomainStates,
+                max_length=100,
+            ),
+        ),
+        migrations.AddField(
+            model_name="domain",
+            name="state_attempted",
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="domain",
+            name="state_changed",
+            field=models.DateTimeField(
+                auto_now_add=True, default=django.utils.timezone.now
+            ),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name="domain",
+            name="state_locked_until",
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="domain",
+            name="state_ready",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddIndex(
+            model_name="domain",
+            index=models.Index(
+                fields=["state", "state_attempted"], name="ix_domain_state_attempted"
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="domain",
+            index=models.Index(
+                condition=models.Q(("state_locked_until__isnull", False)),
+                fields=["state_locked_until", "state"],
+                name="ix_domain_state_locked",
+            ),
+        ),
+    ]

users/models/domain.py

@@ -1,10 +1,46 @@
+import json
 from typing import Optional
 
+import httpx
 import urlman
+from asgiref.sync import sync_to_async
+from django.conf import settings
 from django.db import models
 
+from stator.models import State, StateField, StateGraph, StatorModel
+from users.schemas import NodeInfo
 
-class Domain(models.Model):
+
+class DomainStates(StateGraph):
+    outdated = State(try_interval=60 * 30, force_initial=True)
+    updated = State(try_interval=60 * 60 * 24, attempt_immediately=False)
+    connection_issue = State(externally_progressed=True)
+    purged = State()
+
+    outdated.transitions_to(updated)
+    updated.transitions_to(outdated)
+
+    outdated.transitions_to(connection_issue)
+    outdated.transitions_to(purged)
+    connection_issue.transitions_to(outdated)
+    connection_issue.transitions_to(purged)
+
+    outdated.times_out_to(connection_issue, 60 * 60 * 24)
+
+    @classmethod
+    async def handle_outdated(cls, instance: "Domain"):
+        info = await instance.fetch_nodeinfo()
+        if info:
+            instance.nodeinfo = info.dict()
+            await sync_to_async(instance.save)()
+            return cls.updated
+
+    @classmethod
+    async def handle_updated(cls, instance: "Domain"):
+        return cls.outdated
+
+
+class Domain(StatorModel):
     """
     Represents a domain that a user can have an account on.
 
@@ -31,6 +67,11 @@ class Domain(models.Model):
         unique=True,
     )
 
+    state = StateField(DomainStates)
+
+    # nodeinfo 2.0 detail about the remote server
+    nodeinfo = models.JSONField(null=True, blank=True)
+
     # If we own this domain
     local = models.BooleanField()
 
@@ -104,3 +145,39 @@ class Domain(models.Model):
                     f"Service domain {self.service_domain} is already a domain elsewhere!"
                 )
         super().save(*args, **kwargs)
+
+    async def fetch_nodeinfo(self) -> NodeInfo | None:
+        """
+        Fetch the /NodeInfo/2.0 for the domain
+        """
+        async with httpx.AsyncClient(
+            timeout=settings.SETUP.REMOTE_TIMEOUT,
+            headers={"User-Agent": settings.TAKAHE_USER_AGENT},
+        ) as client:
+            try:
+                response = await client.get(
+                    f"https://{self.domain}/nodeinfo/2.0",
+                    follow_redirects=True,
+                    headers={"Accept": "application/json"},
+                )
+                response.raise_for_status()
+            except httpx.HTTPError as ex:
+                response = getattr(ex, "response", None)
+                if (
+                    response
+                    and response.status_code < 500
+                    and response.status_code not in [401, 403, 404, 410]
+                ):
+                    raise ValueError(
+                        f"Client error fetching nodeinfo: domain={self.domain_id}, code={response.status_code}",
+                        response.content,
+                    )
+                return None
+
+            try:
+                info = NodeInfo(**response.json())
+            except json.JSONDecodeError as ex:
+                raise ValueError(
+                    f"Client error decoding nodeinfo: domain={self.domain_id}, error={str(ex)}"
+                )
+            return info

users/schemas.py

@@ -0,0 +1,32 @@
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class NodeInfoServices(BaseModel):
+    inbound: list[str]
+    outbound: list[str]
+
+
+class NodeInfoSoftware(BaseModel):
+    name: str
+    version: str = "unknown"
+
+
+class NodeInfoUsage(BaseModel):
+    users: dict[str, int] | None
+    local_posts: int = Field(default=0, alias="localPosts")
+
+
+class NodeInfo(BaseModel):
+
+    version: Literal["2.0"]
+    software: NodeInfoSoftware
+    protocols: list[str] | None
+    open_registrations: bool = Field(alias="openRegistrations")
+    usage: NodeInfoUsage
+
+    metadata: dict[str, Any]
+
+    class Config:
+        extra = "ignore"