KayFaraday
/
pleroma-ebooks
1
0
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

markov.py: fix queries to reflect schema changes

This commit is contained in:
io 2021-07-26 06:24:26 +00:00
parent dce92186f9
commit 4e4619fbe0
1 changed files with 22 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
generators/markov.py
View File

@ -8,16 +8,34 @@ def make_sentence(cfg):
def test_sentence_input(self, sentence): def test_sentence_input(self, sentence):
return True # all sentences are valid <3 return True # all sentences are valid <3
db = sqlite3.connect("toots.db") db = sqlite3.connect(cfg.get("db_path", "posts.db"))
db.text_factory = str db.text_factory = str
c = db.cursor() c = db.cursor()
if cfg['learn_from_cw']: if cfg['learn_from_cw']:
ignored_cws_query_params = "(" + ",".join("?" * len(cfg["ignored_cws"])) + ")" ignored_cws_query_params = "(" + ",".join("?" * len(cfg["ignored_cws"])) + ")"
toots = c.execute(f"SELECT content FROM `toots` WHERE cw IS NULL OR CW NOT IN {ignored_cws_query_params} ORDER BY RANDOM() LIMIT 10000", cfg["ignored_cws"]).fetchall() toots = c.execute(
f"""
SELECT content
FROM posts
WHERE
summary IS NULL
OR summary NOT IN {ignored_cws_query_params}
ORDER BY RANDOM() LIMIT 10000
""",
cfg["ignored_cws"],
).fetchall()
else: else:
toots = c.execute("SELECT content FROM `toots` WHERE cw IS NULL ORDER BY RANDOM() LIMIT 10000").fetchall() toots = c.execute(
"""
SELECT content
FROM posts
WHERE summary IS NULL
ORDER BY RANDOM()
LIMIT 10000
""",
).fetchall()
if len(toots) == 0: if not toots:
raise ValueError("Database is empty! Try running main.py.") raise ValueError("Database is empty! Try running main.py.")
nlt = markovify.NewlineText if cfg['overlap_ratio_enabled'] else nlt_fixed nlt = markovify.NewlineText if cfg['overlap_ratio_enabled'] else nlt_fixed