with linebreak for p in soup.select("p"): p.insert_after("\n") p.unwrap() - # keep hashtags in the toots + # fix hashtags for ht in soup.select("a.hashtag"): ht.unwrap() - # unwrap all links (i like the bots posting links) + # fix links for link in soup.select("a"): link.insert_after(link["href"]) link.decompose() - text = map(lambda a: a.strip(), soup.get_text().strip().split("\n")) - - # next up: store this and patch markovify to take it - # return {"text": text, "mentions": mentions, "links": links} - # it's 4am though so we're not doing that now, but i still want the parser updates - #todo: we split above and join now, which is dumb, but i don't wanna mess with the map code bc i don't understand it uwu - text = "\n".join(list(text)) - text = text.replace("'", "'") - return text - -def get_toots(client, id, since_id): - i = 0 - toots = client.account_statuses(id, since_id = since_id) - while toots is not None and len(toots) > 0: - for toot in toots: - t = parse_toot(toot) - if t != None: - yield { - "content": t, - "id": toot.id - } - try: - toots = client.fetch_next(toots) - except TimeoutError: - print("Operation timed out, committing to database and exiting.") - db.commit() - db.close() - sys.exit(1) - i += 1 - if i%10 == 0: - print(i) + toot = soup.get_text() + toot = toot.rstrip("\n") #remove trailing newline + toot = toot.replace("@", "@\u202B") #put a zws between @ and username to avoid mentioning + return(toot) client = Mastodon( - client_id="clientcred.secret", - access_token="usercred.secret", - api_base_url=cfg['site']) + client_id=cfg['client']['id'], + client_secret = cfg['client']['secret'], + access_token=cfg['secret'], + api_base_url=cfg['site']) me = client.account_verify_credentials() following = client.account_following(me.id) @@ -105,7 +95,7 @@ following = client.account_following(me.id) db = sqlite3.connect("toots.db") db.text_factory=str c = db.cursor() -c.execute("CREATE TABLE IF NOT EXISTS `toots` (id INT NOT NULL UNIQUE PRIMARY KEY, userid INT NOT NULL, content VARCHAR NOT NULL) WITHOUT ROWID") +c.execute("CREATE TABLE IF NOT EXISTS `toots` (id INT NOT NULL UNIQUE PRIMARY KEY, userid INT NOT NULL, uri VARCHAR NOT NULL, content VARCHAR NOT NULL) WITHOUT ROWID") db.commit() def handleCtrlC(signal, frame): @@ -121,12 +111,65 @@ for f in following: last_toot = last_toot[0] else: last_toot = 0 - print("Downloading toots for user @{}, starting from {}".format(f.username, last_toot)) - for t in get_toots(client, f.id, last_toot): - # try: - c.execute("REPLACE INTO toots (id, userid, content) VALUES (?, ?, ?)", (t['id'], f.id, t['content'])) - # except: - # pass #ignore toots that can't be encoded properly + print("Harvesting toots for user @{}, starting from {}".format(f.acct, last_toot)) + + #find the user's activitypub outbox + print("WebFingering...") + instance = re.search(r"^.*@(.+)", f.acct) + if instance == None: + instance = re.search(r"https?:\/\/(.*)", cfg['site']).group(1) + else: + instance = instance.group(1) + + # print("{} is on {}".format(f.acct, instance)) + try: + r = requests.get("https://{}/.well-known/host-meta".format(instance)) + uri = re.search(r'template="([^"]+)"', r.text).group(1) + uri = uri.format(uri = "{}@{}".format(f.username, instance)) + r = requests.get(uri) + uri = r.json()['aliases'][1] #TODO: find out if it's safe to rely on this + uri = "{}/outbox?page=true&min_id={}".format(uri, last_toot) + r = requests.get(uri) + j = r.json() + except Exception: + print("oopsy woopsy!! we made a fucky wucky!!!\n(we're probably rate limited, please hang up and try again)") + sys.exit(1) + + print("Downloading and parsing toots", end='', flush=True) + current = None + try: + while len(j['orderedItems']) > 0: + for oi in j['orderedItems']: + if oi['type'] == "Create": + # its a toost baby + content = oi['object']['content'] + if oi['object']['summary'] != None: + #don't download CW'd toots + continue + toot = extract_toot(content) + # print(toot) + try: + c.execute("REPLACE INTO toots (id, userid, uri, content) VALUES (?, ?, ?, ?)", + (re.search(r"[^\/]+$", oi['object']['id']).group(0), + f.id, + oi['object']['id'], + toot + ) + ) + pass + except: + pass #ignore any toots that don't go into the DB + # sys.exit(0) + r = requests.get(j['prev']) + j = r.json() + print('.', end='', flush=True) + print(" Done!") + db.commit() + except: + print("Encountered an error! Saving toots to database and exiting.") + db.commit() + db.close() + sys.exit(1) db.commit() db.execute("VACUUM") #compact db