Get Actor fetching and parsing working

This commit is contained in:
Andrew Godwin 2022-11-05 17:51:54 -06:00
parent 57e33f1215
commit e44a321ec5
13 changed files with 567 additions and 18 deletions

View File

@ -1,6 +1,12 @@
from django.apps import AppConfig
from pyld import jsonld
from core.ld import builtin_document_loader
class CoreConfig(AppConfig):
default_auto_field = "django.db.models.BigAutoField"
name = "core"
def ready(self) -> None:
jsonld.set_document_loader(builtin_document_loader)

302
core/ld.py Normal file
View File

@ -0,0 +1,302 @@
import urllib.parse as urllib_parse
from pyld import jsonld
from pyld.jsonld import JsonLdError
schemas = {
"www.w3.org/ns/activitystreams": {
"contentType": "application/ld+json",
"documentUrl": "https://www.w3.org/ns/activitystreams",
"contextUrl": None,
"document": {
"@context": {
"@vocab": "_:",
"xsd": "http://www.w3.org/2001/XMLSchema#",
"as": "https://www.w3.org/ns/activitystreams#",
"ldp": "http://www.w3.org/ns/ldp#",
"vcard": "http://www.w3.org/2006/vcard/ns#",
"id": "@id",
"type": "@type",
"Accept": "as:Accept",
"Activity": "as:Activity",
"IntransitiveActivity": "as:IntransitiveActivity",
"Add": "as:Add",
"Announce": "as:Announce",
"Application": "as:Application",
"Arrive": "as:Arrive",
"Article": "as:Article",
"Audio": "as:Audio",
"Block": "as:Block",
"Collection": "as:Collection",
"CollectionPage": "as:CollectionPage",
"Relationship": "as:Relationship",
"Create": "as:Create",
"Delete": "as:Delete",
"Dislike": "as:Dislike",
"Document": "as:Document",
"Event": "as:Event",
"Follow": "as:Follow",
"Flag": "as:Flag",
"Group": "as:Group",
"Ignore": "as:Ignore",
"Image": "as:Image",
"Invite": "as:Invite",
"Join": "as:Join",
"Leave": "as:Leave",
"Like": "as:Like",
"Link": "as:Link",
"Mention": "as:Mention",
"Note": "as:Note",
"Object": "as:Object",
"Offer": "as:Offer",
"OrderedCollection": "as:OrderedCollection",
"OrderedCollectionPage": "as:OrderedCollectionPage",
"Organization": "as:Organization",
"Page": "as:Page",
"Person": "as:Person",
"Place": "as:Place",
"Profile": "as:Profile",
"Question": "as:Question",
"Reject": "as:Reject",
"Remove": "as:Remove",
"Service": "as:Service",
"TentativeAccept": "as:TentativeAccept",
"TentativeReject": "as:TentativeReject",
"Tombstone": "as:Tombstone",
"Undo": "as:Undo",
"Update": "as:Update",
"Video": "as:Video",
"View": "as:View",
"Listen": "as:Listen",
"Read": "as:Read",
"Move": "as:Move",
"Travel": "as:Travel",
"IsFollowing": "as:IsFollowing",
"IsFollowedBy": "as:IsFollowedBy",
"IsContact": "as:IsContact",
"IsMember": "as:IsMember",
"subject": {"@id": "as:subject", "@type": "@id"},
"relationship": {"@id": "as:relationship", "@type": "@id"},
"actor": {"@id": "as:actor", "@type": "@id"},
"attributedTo": {"@id": "as:attributedTo", "@type": "@id"},
"attachment": {"@id": "as:attachment", "@type": "@id"},
"bcc": {"@id": "as:bcc", "@type": "@id"},
"bto": {"@id": "as:bto", "@type": "@id"},
"cc": {"@id": "as:cc", "@type": "@id"},
"context": {"@id": "as:context", "@type": "@id"},
"current": {"@id": "as:current", "@type": "@id"},
"first": {"@id": "as:first", "@type": "@id"},
"generator": {"@id": "as:generator", "@type": "@id"},
"icon": {"@id": "as:icon", "@type": "@id"},
"image": {"@id": "as:image", "@type": "@id"},
"inReplyTo": {"@id": "as:inReplyTo", "@type": "@id"},
"items": {"@id": "as:items", "@type": "@id"},
"instrument": {"@id": "as:instrument", "@type": "@id"},
"orderedItems": {
"@id": "as:items",
"@type": "@id",
"@container": "@list",
},
"last": {"@id": "as:last", "@type": "@id"},
"location": {"@id": "as:location", "@type": "@id"},
"next": {"@id": "as:next", "@type": "@id"},
"object": {"@id": "as:object", "@type": "@id"},
"oneOf": {"@id": "as:oneOf", "@type": "@id"},
"anyOf": {"@id": "as:anyOf", "@type": "@id"},
"closed": {"@id": "as:closed", "@type": "xsd:dateTime"},
"origin": {"@id": "as:origin", "@type": "@id"},
"accuracy": {"@id": "as:accuracy", "@type": "xsd:float"},
"prev": {"@id": "as:prev", "@type": "@id"},
"preview": {"@id": "as:preview", "@type": "@id"},
"replies": {"@id": "as:replies", "@type": "@id"},
"result": {"@id": "as:result", "@type": "@id"},
"audience": {"@id": "as:audience", "@type": "@id"},
"partOf": {"@id": "as:partOf", "@type": "@id"},
"tag": {"@id": "as:tag", "@type": "@id"},
"target": {"@id": "as:target", "@type": "@id"},
"to": {"@id": "as:to", "@type": "@id"},
"url": {"@id": "as:url", "@type": "@id"},
"altitude": {"@id": "as:altitude", "@type": "xsd:float"},
"content": "as:content",
"contentMap": {"@id": "as:content", "@container": "@language"},
"name": "as:name",
"nameMap": {"@id": "as:name", "@container": "@language"},
"duration": {"@id": "as:duration", "@type": "xsd:duration"},
"endTime": {"@id": "as:endTime", "@type": "xsd:dateTime"},
"height": {"@id": "as:height", "@type": "xsd:nonNegativeInteger"},
"href": {"@id": "as:href", "@type": "@id"},
"hreflang": "as:hreflang",
"latitude": {"@id": "as:latitude", "@type": "xsd:float"},
"longitude": {"@id": "as:longitude", "@type": "xsd:float"},
"mediaType": "as:mediaType",
"published": {"@id": "as:published", "@type": "xsd:dateTime"},
"radius": {"@id": "as:radius", "@type": "xsd:float"},
"rel": "as:rel",
"startIndex": {
"@id": "as:startIndex",
"@type": "xsd:nonNegativeInteger",
},
"startTime": {"@id": "as:startTime", "@type": "xsd:dateTime"},
"summary": "as:summary",
"summaryMap": {"@id": "as:summary", "@container": "@language"},
"totalItems": {
"@id": "as:totalItems",
"@type": "xsd:nonNegativeInteger",
},
"units": "as:units",
"updated": {"@id": "as:updated", "@type": "xsd:dateTime"},
"width": {"@id": "as:width", "@type": "xsd:nonNegativeInteger"},
"describes": {"@id": "as:describes", "@type": "@id"},
"formerType": {"@id": "as:formerType", "@type": "@id"},
"deleted": {"@id": "as:deleted", "@type": "xsd:dateTime"},
"inbox": {"@id": "ldp:inbox", "@type": "@id"},
"outbox": {"@id": "as:outbox", "@type": "@id"},
"following": {"@id": "as:following", "@type": "@id"},
"followers": {"@id": "as:followers", "@type": "@id"},
"streams": {"@id": "as:streams", "@type": "@id"},
"preferredUsername": "as:preferredUsername",
"endpoints": {"@id": "as:endpoints", "@type": "@id"},
"uploadMedia": {"@id": "as:uploadMedia", "@type": "@id"},
"proxyUrl": {"@id": "as:proxyUrl", "@type": "@id"},
"liked": {"@id": "as:liked", "@type": "@id"},
"oauthAuthorizationEndpoint": {
"@id": "as:oauthAuthorizationEndpoint",
"@type": "@id",
},
"oauthTokenEndpoint": {"@id": "as:oauthTokenEndpoint", "@type": "@id"},
"provideClientKey": {"@id": "as:provideClientKey", "@type": "@id"},
"signClientKey": {"@id": "as:signClientKey", "@type": "@id"},
"sharedInbox": {"@id": "as:sharedInbox", "@type": "@id"},
"Public": {"@id": "as:Public", "@type": "@id"},
"source": "as:source",
"likes": {"@id": "as:likes", "@type": "@id"},
"shares": {"@id": "as:shares", "@type": "@id"},
"alsoKnownAs": {"@id": "as:alsoKnownAs", "@type": "@id"},
}
},
},
"w3id.org/security/v1": {
"contentType": "application/ld+json",
"documentUrl": "https://w3id.org/security/v1",
"contextUrl": None,
"document": {
"@context": {
"id": "@id",
"type": "@type",
"dc": "http://purl.org/dc/terms/",
"sec": "https://w3id.org/security#",
"xsd": "http://www.w3.org/2001/XMLSchema#",
"EcdsaKoblitzSignature2016": "sec:EcdsaKoblitzSignature2016",
"Ed25519Signature2018": "sec:Ed25519Signature2018",
"EncryptedMessage": "sec:EncryptedMessage",
"GraphSignature2012": "sec:GraphSignature2012",
"LinkedDataSignature2015": "sec:LinkedDataSignature2015",
"LinkedDataSignature2016": "sec:LinkedDataSignature2016",
"CryptographicKey": "sec:Key",
"authenticationTag": "sec:authenticationTag",
"canonicalizationAlgorithm": "sec:canonicalizationAlgorithm",
"cipherAlgorithm": "sec:cipherAlgorithm",
"cipherData": "sec:cipherData",
"cipherKey": "sec:cipherKey",
"created": {"@id": "dc:created", "@type": "xsd:dateTime"},
"creator": {"@id": "dc:creator", "@type": "@id"},
"digestAlgorithm": "sec:digestAlgorithm",
"digestValue": "sec:digestValue",
"domain": "sec:domain",
"encryptionKey": "sec:encryptionKey",
"expiration": {"@id": "sec:expiration", "@type": "xsd:dateTime"},
"expires": {"@id": "sec:expiration", "@type": "xsd:dateTime"},
"initializationVector": "sec:initializationVector",
"iterationCount": "sec:iterationCount",
"nonce": "sec:nonce",
"normalizationAlgorithm": "sec:normalizationAlgorithm",
"owner": {"@id": "sec:owner", "@type": "@id"},
"password": "sec:password",
"privateKey": {"@id": "sec:privateKey", "@type": "@id"},
"privateKeyPem": "sec:privateKeyPem",
"publicKey": {"@id": "sec:publicKey", "@type": "@id"},
"publicKeyBase58": "sec:publicKeyBase58",
"publicKeyPem": "sec:publicKeyPem",
"publicKeyWif": "sec:publicKeyWif",
"publicKeyService": {"@id": "sec:publicKeyService", "@type": "@id"},
"revoked": {"@id": "sec:revoked", "@type": "xsd:dateTime"},
"salt": "sec:salt",
"signature": "sec:signature",
"signatureAlgorithm": "sec:signingAlgorithm",
"signatureValue": "sec:signatureValue",
}
},
},
}
def builtin_document_loader(url: str, options={}):
# Get URL without scheme
pieces = urllib_parse.urlparse(url)
if pieces.hostname is None:
raise JsonLdError(
f"No schema built-in for {url!r}",
"jsonld.LoadDocumentError",
code="loading document failed",
cause="NoHostnameError",
)
key = pieces.hostname + pieces.path.rstrip("/")
try:
return schemas[key]
except KeyError:
raise JsonLdError(
f"No schema built-in for {key!r}",
"jsonld.LoadDocumentError",
code="loading document failed",
cause="KeyError",
)
class LDDocument:
"""
Utility class for dealing with a document a bit more easily
"""
def __init__(self, json_data):
self.items = {}
for entry in jsonld.flatten(jsonld.expand(json_data)):
item = LDItem(self, entry)
self.items[item.id] = item
def by_type(self, type):
for item in self.items.values():
if item.type == type:
yield item
class LDItem:
"""
Represents a single item in an LDDocument
"""
def __init__(self, document, data):
self.data = data
self.document = document
self.id = self.data["@id"]
if "@type" in self.data:
self.type = self.data["@type"][0]
else:
self.type = None
def get(self, key):
"""
Gets the first value of the given key, or None if it's not present.
If it's an ID reference, returns the other Item if possible, or the raw
ID if it's not supplied.
"""
contents = self.data.get(key)
if not contents:
return None
id = contents[0].get("@id")
value = contents[0].get("@value")
if value is not None:
return value
if id in self.document.items:
return self.document.items[id]
else:
return id

View File

@ -4,3 +4,4 @@ pillow~=9.3.0
urlman~=2.0.1
django-crispy-forms~=1.14
cryptography~=38.0
httpx~=0.23

View File

@ -1,4 +1,4 @@
# Generated by Django 4.1.3 on 2022-11-05 19:43
# Generated by Django 4.1.3 on 2022-11-05 23:50
import django.db.models.deletion
from django.db import migrations, models

View File

@ -1,3 +1,4 @@
import urlman
from django.db import models
@ -33,3 +34,6 @@ class Status(models.Model):
text=text,
local=True,
)
class urls(urlman.Urls):
view = "{self.identity.urls.view}{self.id}/"

View File

@ -12,11 +12,12 @@ urlpatterns = [
# Identity views
path("@<handle>/", identity.ViewIdentity.as_view()),
path("@<handle>/actor/", identity.Actor.as_view()),
path("@<handle>/actor/inbox/", identity.Inbox.as_view()),
# Identity selection
path("identity/select/", identity.SelectIdentity.as_view()),
path("identity/create/", identity.CreateIdentity.as_view()),
# Well-known endpoints
path(".well-known/webfinger/", identity.Webfinger.as_view()),
path(".well-known/webfinger", identity.Webfinger.as_view()),
# Django admin
path("djadmin/", admin.site.urls),
]

View File

@ -5,6 +5,8 @@
<small>{{ status.identity.short_handle }}</small>
</a>
</h3>
<time>{{ status.created | timesince }} ago</time>
<time>
<a href="{{ status.urls.view }}">{{ status.created | timesince }} ago</a>
</time>
{{ status.text | linebreaks }}
</div>

View File

@ -1,4 +1,4 @@
# Generated by Django 4.1.3 on 2022-11-05 19:15
# Generated by Django 4.1.3 on 2022-11-05 23:50
import functools
@ -96,32 +96,50 @@ class Migration(migrations.Migration):
),
("handle", models.CharField(max_length=500, unique=True)),
("name", models.CharField(blank=True, max_length=500, null=True)),
("bio", models.TextField(blank=True, null=True)),
("summary", models.TextField(blank=True, null=True)),
("actor_uri", models.CharField(blank=True, max_length=500, null=True)),
(
"profile_image",
"profile_uri",
models.CharField(blank=True, max_length=500, null=True),
),
("inbox_uri", models.CharField(blank=True, max_length=500, null=True)),
("outbox_uri", models.CharField(blank=True, max_length=500, null=True)),
("icon_uri", models.CharField(blank=True, max_length=500, null=True)),
("image_uri", models.CharField(blank=True, max_length=500, null=True)),
(
"icon",
models.ImageField(
blank=True,
null=True,
upload_to=functools.partial(
users.models.identity.upload_namer,
*("profile_images",),
**{},
)
),
),
),
(
"background_image",
"image",
models.ImageField(
blank=True,
null=True,
upload_to=functools.partial(
users.models.identity.upload_namer,
*("background_images",),
**{},
)
),
),
),
("local", models.BooleanField()),
("private_key", models.BinaryField(blank=True, null=True)),
("public_key", models.BinaryField(blank=True, null=True)),
(
"manually_approves_followers",
models.BooleanField(blank=True, null=True),
),
("private_key", models.TextField(blank=True, null=True)),
("public_key", models.TextField(blank=True, null=True)),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
("fetched", models.DateTimeField(blank=True, null=True)),
("deleted", models.DateTimeField(blank=True, null=True)),
(
"users",
@ -131,4 +149,37 @@ class Migration(migrations.Migration):
),
],
),
migrations.CreateModel(
name="Follow",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("note", models.TextField(blank=True, null=True)),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
(
"source",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="outbound_follows",
to="users.identity",
),
),
(
"target",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="inbound_follows",
to="users.identity",
),
),
],
),
]

View File

@ -1,3 +1,4 @@
from .follow import Follow # noqa
from .identity import Identity # noqa
from .user import User # noqa
from .user_event import UserEvent # noqa

23
users/models/follow.py Normal file
View File

@ -0,0 +1,23 @@
from django.db import models
class Follow(models.Model):
"""
Tracks major events that happen to users
"""
source = models.ForeignKey(
"users.Identity",
on_delete=models.CASCADE,
related_name="outbound_follows",
)
target = models.ForeignKey(
"users.Identity",
on_delete=models.CASCADE,
related_name="inbound_follows",
)
note = models.TextField(blank=True, null=True)
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)

View File

@ -2,12 +2,17 @@ import base64
import uuid
from functools import partial
import httpx
import urlman
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from asgiref.sync import sync_to_async
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import padding, rsa
from django.conf import settings
from django.db import models
from django.utils import timezone
from django.utils.http import http_date
from core.ld import LDDocument
def upload_namer(prefix, instance, filename):
@ -27,20 +32,31 @@ class Identity(models.Model):
# The handle includes the domain!
handle = models.CharField(max_length=500, unique=True)
name = models.CharField(max_length=500, blank=True, null=True)
bio = models.TextField(blank=True, null=True)
summary = models.TextField(blank=True, null=True)
profile_image = models.ImageField(upload_to=partial(upload_namer, "profile_images"))
background_image = models.ImageField(
upload_to=partial(upload_namer, "background_images")
actor_uri = models.CharField(max_length=500, blank=True, null=True)
profile_uri = models.CharField(max_length=500, blank=True, null=True)
inbox_uri = models.CharField(max_length=500, blank=True, null=True)
outbox_uri = models.CharField(max_length=500, blank=True, null=True)
icon_uri = models.CharField(max_length=500, blank=True, null=True)
image_uri = models.CharField(max_length=500, blank=True, null=True)
icon = models.ImageField(
upload_to=partial(upload_namer, "profile_images"), blank=True, null=True
)
image = models.ImageField(
upload_to=partial(upload_namer, "background_images"), blank=True, null=True
)
local = models.BooleanField()
users = models.ManyToManyField("users.User", related_name="identities")
manually_approves_followers = models.BooleanField(blank=True, null=True)
private_key = models.TextField(null=True, blank=True)
public_key = models.TextField(null=True, blank=True)
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
fetched = models.DateTimeField(null=True, blank=True)
deleted = models.DateTimeField(null=True, blank=True)
@property
@ -69,6 +85,128 @@ class Identity(models.Model):
)
self.save()
async def fetch_details(self):
if self.local:
raise ValueError("Cannot fetch local identities")
self.actor_uri = None
self.inbox_uri = None
self.profile_uri = None
# Go knock on webfinger and see what their address is
await self.fetch_webfinger()
# Fetch actor JSON
if self.actor_uri:
await self.fetch_actor()
self.fetched = timezone.now()
await sync_to_async(self.save)()
async def fetch_webfinger(self) -> bool:
async with httpx.AsyncClient() as client:
response = await client.get(
f"https://{self.domain}/.well-known/webfinger?resource=acct:{self.handle}",
headers={"Accept": "application/json"},
)
if response.status_code >= 400:
return False
data = response.json()
for link in data["links"]:
if (
link.get("type") == "application/activity+json"
and link.get("rel") == "self"
):
self.actor_uri = link["href"]
elif (
link.get("type") == "text/html"
and link.get("rel") == "http://webfinger.net/rel/profile-page"
):
self.profile_uri = link["href"]
return True
async def fetch_actor(self) -> bool:
async with httpx.AsyncClient() as client:
response = await client.get(
self.actor_uri,
headers={"Accept": "application/json"},
)
if response.status_code >= 400:
return False
data = response.json()
document = LDDocument(data)
for person in document.by_type(
"https://www.w3.org/ns/activitystreams#Person"
):
self.name = person.get("https://www.w3.org/ns/activitystreams#name")
self.summary = person.get(
"https://www.w3.org/ns/activitystreams#summary"
)
self.inbox_uri = person.get("http://www.w3.org/ns/ldp#inbox")
self.outbox_uri = person.get(
"https://www.w3.org/ns/activitystreams#outbox"
)
self.manually_approves_followers = person.get(
"https://www.w3.org/ns/activitystreams#manuallyApprovesFollowers'"
)
self.private_key = person.get(
"https://w3id.org/security#publicKey"
).get("https://w3id.org/security#publicKeyPem")
icon = person.get("https://www.w3.org/ns/activitystreams#icon")
if icon:
self.icon_uri = icon.get(
"https://www.w3.org/ns/activitystreams#url"
)
image = person.get("https://www.w3.org/ns/activitystreams#image")
if image:
self.image_uri = image.get(
"https://www.w3.org/ns/activitystreams#url"
)
return True
async def signed_request(self, host, method, path, document):
"""
Delivers the document to the specified host, method, path and signed
as this user.
"""
private_key = serialization.load_pem_private_key(
self.private_key,
password=None,
)
date_string = http_date(timezone.now().timestamp())
headers = {
"(request-target)": f"{method} {path}",
"Host": host,
"Date": date_string,
}
headers_string = " ".join(headers.keys())
signed_string = "\n".join(f"{name}: {value}" for name, value in headers.items())
signature = base64.b64encode(
private_key.sign(
signed_string,
padding.PSS(
mgf=padding.MGF1(hashes.SHA256()),
salt_length=padding.PSS.MAX_LENGTH,
),
hashes.SHA256(),
)
)
del headers["(request-target)"]
headers[
"Signature"
] = f'keyId="https://{settings.DEFAULT_DOMAIN}{self.urls.actor}",headers="{headers_string}",signature="{signature}"'
async with httpx.AsyncClient() as client:
return await client.request(
method,
"https://{host}{path}",
headers=headers,
data=document,
)
def validate_signature(self, request):
"""
Attempts to validate the signature on an incoming request.
Returns False if the signature is invalid, None if it cannot be verified
as we do not have the key locally, or the name of the actor if it is valid.
"""
pass
def __str__(self):
return self.name or self.handle

View File

@ -56,3 +56,9 @@ class User(AbstractBaseUser):
@property
def is_staff(self):
return self.admin
def has_module_perms(self, module):
return self.admin
def has_perm(self, perm):
return self.admin

View File

@ -4,6 +4,7 @@ from django.contrib.auth.decorators import login_required
from django.http import Http404, JsonResponse
from django.shortcuts import redirect
from django.utils.decorators import method_decorator
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import FormView, TemplateView, View
from core.forms import FormHelper
@ -88,7 +89,7 @@ class Actor(View):
],
"id": f"https://{settings.DEFAULT_DOMAIN}{identity.urls.actor}",
"type": "Person",
"preferredUsername": "alice",
"preferredUsername": identity.short_handle,
"inbox": f"https://{settings.DEFAULT_DOMAIN}{identity.urls.inbox}",
"publicKey": {
"id": f"https://{settings.DEFAULT_DOMAIN}{identity.urls.actor}#main-key",
@ -99,6 +100,19 @@ class Actor(View):
)
@method_decorator(csrf_exempt, name="dispatch")
class Inbox(View):
"""
AP Inbox endpoint
"""
def post(self, request, handle):
# Validate the signature
signature = request.META.get("HTTP_SIGNATURE")
print(signature)
print(request.body)
class Webfinger(View):
"""
Services webfinger requests