pleroma-ebooks/main.py

286 lines
10 KiB
Python
Raw Normal View History

2018-10-08 18:11:51 -07:00
#!/usr/bin/env python3
# toot downloader version two!!
2018-10-08 18:11:51 -07:00
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from mastodon import Mastodon, MastodonUnauthorizedError
2018-10-08 18:11:51 -07:00
from os import path
from bs4 import BeautifulSoup
import os, sqlite3, signal, sys, json, re, shutil, argparse
import requests
import functions
2018-10-08 18:11:51 -07:00
parser = argparse.ArgumentParser(description='Log in and download posts.')
2019-08-06 20:48:45 -07:00
parser.add_argument('-c', '--cfg', dest='cfg', default='config.json', nargs='?',
help="Specify a custom location for config.json.")
args = parser.parse_args()
scopes = ["read:statuses", "read:accounts", "read:follows", "write:statuses", "read:notifications", "write:accounts"]
2019-04-28 20:59:37 -07:00
#cfg defaults
2019-04-28 20:59:37 -07:00
cfg = {
"site": "https://botsin.space",
"cw": None,
"instance_blacklist": ["bofa.lol", "witches.town", "knzk.me"], # rest in piece
2019-04-28 21:24:52 -07:00
"learn_from_cw": False,
"mention_handling": 1,
"max_thread_length": 15,
"strip_paired_punctuation": False
2019-04-28 20:59:37 -07:00
}
2019-08-13 22:00:35 -07:00
try:
cfg.update(json.load(open(args.cfg, 'r')))
except FileNotFoundError:
open(args.cfg, "w").write("{}")
print("Using {} as configuration file".format(args.cfg))
if not cfg['site'].startswith("https://") and not cfg['site'].startswith("http://"):
print("Site must begin with 'https://' or 'http://'. Value '{}' is invalid - try 'https://{}' instead.".format(cfg['site']))
sys.exit(1)
if "client" not in cfg:
2019-01-11 04:08:10 -08:00
print("No application info -- registering application with {}".format(cfg['site']))
client_id, client_secret = Mastodon.create_app("mstdn-ebooks",
api_base_url=cfg['site'],
scopes=scopes,
website="https://github.com/Lynnesbian/mstdn-ebooks")
cfg['client'] = {
"id": client_id,
"secret": client_secret
}
if "secret" not in cfg:
2019-01-11 04:08:10 -08:00
print("No user credentials -- logging in to {}".format(cfg['site']))
client = Mastodon(client_id = cfg['client']['id'],
client_secret = cfg['client']['secret'],
api_base_url=cfg['site'])
2018-10-08 18:11:51 -07:00
2019-01-11 04:08:10 -08:00
print("Open this URL and authenticate to give mstdn-ebooks access to your bot's account: {}".format(client.auth_request_url(scopes=scopes)))
cfg['secret'] = client.log_in(code=input("Secret: "), scopes=scopes)
2018-10-08 18:11:51 -07:00
json.dump(cfg, open(args.cfg, "w+"))
2018-10-08 18:11:51 -07:00
def extract_toot(toot):
toot = functions.extract_toot(toot)
2018-11-27 03:29:50 -08:00
toot = toot.replace("@", "@\u200B") #put a zws between @ and username to avoid mentioning
return(toot)
2018-10-08 18:11:51 -07:00
client = Mastodon(
client_id=cfg['client']['id'],
client_secret = cfg['client']['secret'],
access_token=cfg['secret'],
api_base_url=cfg['site'])
2018-10-08 18:11:51 -07:00
try:
me = client.account_verify_credentials()
except MastodonUnauthorizedError:
print("The provided access token in {} is invalid. Please delete {} and run main.py again.".format(args.cfg, args.cfg))
sys.exit(1)
2018-10-08 18:11:51 -07:00
following = client.account_following(me.id)
db = sqlite3.connect("toots.db")
db.text_factory=str
c = db.cursor()
2019-08-15 09:02:06 -07:00
c.execute("CREATE TABLE IF NOT EXISTS `toots` (sortid INTEGER UNIQUE PRIMARY KEY AUTOINCREMENT, id VARCHAR NOT NULL, cw INT NOT NULL DEFAULT 0, userid VARCHAR NOT NULL, uri VARCHAR NOT NULL, content VARCHAR NOT NULL)")
c.execute("CREATE TRIGGER IF NOT EXISTS `dedup` AFTER INSERT ON toots FOR EACH ROW BEGIN DELETE FROM toots WHERE rowid NOT IN (SELECT MIN(sortid) FROM toots GROUP BY uri ); END; ")
2020-03-08 01:46:07 -08:00
db.commit()
tableinfo = c.execute("PRAGMA table_info(`toots`)").fetchall()
found = False
columns = []
for entry in tableinfo:
if entry[1] == "sortid":
found = True
break
columns.append(entry[1])
if not found:
print("Migrating to new database format. Please wait...")
print("WARNING: If any of the accounts your bot is following are Pleroma users, please delete toots.db and run main.py again to create it anew.")
try:
c.execute("DROP TABLE `toots_temp`")
except:
pass
c.execute("CREATE TABLE `toots_temp` (sortid INTEGER UNIQUE PRIMARY KEY AUTOINCREMENT, id VARCHAR NOT NULL, cw INT NOT NULL DEFAULT 0, userid VARCHAR NOT NULL, uri VARCHAR NOT NULL, content VARCHAR NOT NULL)")
for f in following:
user_toots = c.execute("SELECT * FROM `toots` WHERE userid LIKE ? ORDER BY id", (f.id,)).fetchall()
if user_toots == None:
continue
if columns[-1] == "cw":
for toot in user_toots:
c.execute("INSERT INTO `toots_temp` (id, userid, uri, content, cw) VALUES (?, ?, ?, ?, ?)", toot)
else:
for toot in user_toots:
c.execute("INSERT INTO `toots_temp` (id, cw, userid, uri, content) VALUES (?, ?, ?, ?, ?)", toot)
c.execute("DROP TABLE `toots`")
c.execute("ALTER TABLE `toots_temp` RENAME TO `toots`")
c.execute("CREATE TRIGGER IF NOT EXISTS `dedup` AFTER INSERT ON toots FOR EACH ROW BEGIN DELETE FROM toots WHERE rowid NOT IN (SELECT MIN(sortid) FROM toots GROUP BY uri ); END; ")
2020-03-08 01:46:07 -08:00
2018-10-08 18:11:51 -07:00
db.commit()
def handleCtrlC(signal, frame):
print("\nPREMATURE EVACUATION - Saving chunks")
db.commit()
sys.exit(1)
signal.signal(signal.SIGINT, handleCtrlC)
patterns = {
2019-02-07 07:45:44 -08:00
"handle": re.compile(r"^.*@(.+)"),
"url": re.compile(r"https?:\/\/(.*)"),
"uri": re.compile(r'template="([^"]+)"'),
"pid": re.compile(r"[^\/]+$"),
}
2018-10-27 01:28:20 -07:00
def insert_toot(oii, acc, post, cursor): # extracted to prevent duplication
pid = patterns["pid"].search(oii['object']['id']).group(0)
cursor.execute("REPLACE INTO toots (id, cw, userid, uri, content) VALUES (?, ?, ?, ?, ?)", (
pid,
1 if (oii['object']['summary'] != None and oii['object']['summary'] != "") else 0,
acc.id,
oii['object']['id'],
post
))
2018-10-08 18:11:51 -07:00
for f in following:
last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY sortid DESC LIMIT 1", (f.id,)).fetchone()
2018-10-08 18:11:51 -07:00
if last_toot != None:
last_toot = last_toot[0]
else:
last_toot = 0
2019-05-19 06:06:31 -07:00
print("Downloading posts for user @{}, starting from {}".format(f.acct, last_toot))
#find the user's activitypub outbox
2019-02-24 17:18:38 -08:00
print("WebFingering...")
2019-02-07 07:45:44 -08:00
instance = patterns["handle"].search(f.acct)
if instance == None:
2019-02-07 07:45:44 -08:00
instance = patterns["url"].search(cfg['site']).group(1)
else:
instance = instance.group(1)
2019-01-11 05:08:53 -08:00
if instance in cfg['instance_blacklist']:
print("skipping blacklisted instance: {}".format(instance))
2018-10-25 07:33:57 -07:00
continue
2019-01-11 04:15:05 -08:00
try:
2019-06-30 18:21:08 -07:00
# 1. download host-meta to find webfinger URL
2018-10-27 05:07:38 -07:00
r = requests.get("https://{}/.well-known/host-meta".format(instance), timeout=10)
2019-05-19 06:06:31 -07:00
# 2. use webfinger to find user's info page
2019-02-07 07:45:44 -08:00
uri = patterns["uri"].search(r.text).group(1)
uri = uri.format(uri = "{}@{}".format(f.username, instance))
2018-10-27 05:07:38 -07:00
r = requests.get(uri, headers={"Accept": "application/json"}, timeout=10)
2018-10-27 01:28:20 -07:00
j = r.json()
2019-05-19 06:06:31 -07:00
found = False
for link in j['links']:
if link['rel'] == 'self':
#this is a link formatted like "https://instan.ce/users/username", which is what we need
uri = link['href']
2019-05-19 06:06:31 -07:00
found = True
break
if not found:
print("Couldn't find a valid ActivityPub outbox URL.")
# 3. download first page of outbox
2018-11-06 21:39:12 -08:00
uri = "{}/outbox?page=true".format(uri)
2019-05-19 06:06:31 -07:00
r = requests.get(uri, timeout=15)
j = r.json()
2019-05-19 06:06:31 -07:00
except:
print("oopsy woopsy!! we made a fucky wucky!!!\n(we're probably rate limited, please hang up and try again)")
sys.exit(1)
2018-10-27 01:28:20 -07:00
pleroma = False
if 'next' not in j and 'prev' not in j:
# there's only one page of results, don't bother doing anything special
pass
elif 'prev' not in j:
2019-05-19 06:06:31 -07:00
print("Using Pleroma compatibility mode")
2018-10-27 01:28:20 -07:00
pleroma = True
if 'first' in j:
# apparently there used to be a 'first' field in pleroma's outbox output, but it's not there any more
# i'll keep this for backwards compatibility with older pleroma instances
# it was removed in pleroma 1.0.7 - https://git.pleroma.social/pleroma/pleroma/-/blob/841e4e4d835b8d1cecb33102356ca045571ef1fc/CHANGELOG.md#107-2019-09-26
j = j['first']
else:
2019-05-19 06:06:31 -07:00
print("Using standard mode")
uri = "{}&min_id={}".format(uri, last_toot)
2018-11-06 21:39:12 -08:00
r = requests.get(uri)
j = r.json()
2019-05-19 06:06:31 -07:00
print("Downloading and saving posts", end='', flush=True)
2018-11-06 21:39:12 -08:00
done = False
2018-11-09 03:50:36 -08:00
try:
2018-11-06 21:39:12 -08:00
while not done and len(j['orderedItems']) > 0:
for oi in j['orderedItems']:
2018-11-28 11:36:05 -08:00
if oi['type'] != "Create":
2019-01-11 04:15:05 -08:00
continue #this isn't a toot/post/status/whatever, it's a boost or a follow or some other activitypub thing. ignore
2018-11-28 11:36:05 -08:00
# its a toost baby
content = oi['object']['content']
toot = extract_toot(content)
# print(toot)
try:
if pleroma:
if c.execute("SELECT COUNT(*) FROM toots WHERE uri LIKE ?", (oi['object']['id'],)).fetchone()[0] > 0:
2018-11-28 11:36:05 -08:00
#we've caught up to the notices we've already downloaded, so we can stop now
2019-05-06 10:05:02 -07:00
#you might be wondering, "lynne, what if the instance ratelimits you after 40 posts, and they've made 60 since main.py was last run? wouldn't the bot miss 20 posts and never be able to see them?" to which i reply, "i know but i don't know how to fix it"
2018-11-28 11:36:05 -08:00
done = True
2019-08-15 20:50:47 -07:00
continue
2019-07-09 17:43:56 -07:00
if 'lang' in cfg:
2019-05-06 10:14:30 -07:00
try:
2019-05-19 06:06:31 -07:00
if oi['object']['contentMap'][cfg['lang']]: # filter for language
2019-05-06 10:14:30 -07:00
insert_toot(oi, f, toot, c)
except KeyError:
#JSON doesn't have contentMap, just insert the toot irregardlessly
insert_toot(oi, f, toot, c)
else:
insert_toot(oi, f, toot, c)
2018-11-28 11:36:05 -08:00
pass
except:
pass #ignore any toots that don't successfully go into the DB
2019-05-19 06:06:31 -07:00
# get the next/previous page
try:
if not pleroma:
r = requests.get(j['prev'], timeout=15)
else:
r = requests.get(j['next'], timeout=15)
except requests.Timeout:
print("HTTP timeout, site did not respond within 15 seconds")
except KeyError:
print("Couldn't get next page - we've probably got all the posts")
2019-05-19 06:06:31 -07:00
except:
print("An error occurred while trying to obtain more posts.")
2018-11-06 21:39:12 -08:00
j = r.json()
print('.', end='', flush=True)
print(" Done!")
db.commit()
except requests.HTTPError as e:
if e.response.status_code == 429:
print("Rate limit exceeded. This means we're downloading too many posts in quick succession. Saving toots to database and moving to next followed account.")
db.commit()
else:
# TODO: remove duplicate code
2019-05-19 06:06:31 -07:00
print("Encountered an error! Saving posts to database and moving to next followed account.")
db.commit()
2018-11-09 03:50:36 -08:00
except:
2019-05-19 06:06:31 -07:00
print("Encountered an error! Saving posts to database and moving to next followed account.")
2018-11-09 03:50:36 -08:00
db.commit()
print("Done!")
2018-10-08 18:11:51 -07:00
db.commit()
db.execute("VACUUM") #compact db
db.commit()
db.close()