From 2703a886bd95d2c59ae4c1043fb2f0cb67fa8e92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Charlotte=20=F0=9F=A6=9D=20Delenk?=
Date: Thu, 3 Nov 2022 10:06:02 +0100
Subject: [PATCH] add moa patch
---
moa/default.nix | 12 +-
moa/moa.patch | 1189 +++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 1197 insertions(+), 4 deletions(-)
create mode 100644 moa/moa.patch
diff --git a/moa/default.nix b/moa/default.nix
index 594c752..bcfe51c 100644
--- a/moa/default.nix
+++ b/moa/default.nix
@@ -23,12 +23,16 @@
lib,
setuptools,
psycopg2,
+ applyPatches,
}: let
source = builtins.fromJSON (builtins.readFile ./source.json);
- src = fetchFromGitLab {
- owner = "hexchen";
- repo = "moa";
- inherit (source) rev sha256;
+ src = applyPatches {
+ src = fetchFromGitLab {
+ owner = "hexchen";
+ repo = "moa";
+ inherit (source) rev sha256;
+ };
+ patches = [./moa.patch];
};
moa-env = python3.withPackages (_: [
certifi
diff --git a/moa/moa.patch b/moa/moa.patch
new file mode 100644
index 0000000..ab1e9b5
--- /dev/null
+++ b/moa/moa.patch
@@ -0,0 +1,1189 @@
+diff --git a/.envrc b/.envrc
+new file mode 100644
+index 0000000..3550a30
+--- /dev/null
++++ b/.envrc
+@@ -0,0 +1 @@
++use flake
+diff --git a/.gitignore b/.gitignore
+index b8c6716..4dbdfbb 100644
+--- a/.gitignore
++++ b/.gitignore
+@@ -19,3 +19,4 @@ worker_*.lock
+ .idea/moa.iml
+ Pipfile
+ Pipefile.lock
++/.direnv
+diff --git a/.vscode/settings.json b/.vscode/settings.json
+new file mode 100644
+index 0000000..30e36e9
+--- /dev/null
++++ b/.vscode/settings.json
+@@ -0,0 +1,3 @@
++{
++ "python.formatting.provider": "yapf"
++}
+diff --git a/app.py b/app.py
+index 8a53602..bdaf082 100644
+--- a/app.py
++++ b/app.py
+@@ -3,12 +3,12 @@ import os
+ import random
+ from datetime import datetime, timedelta
+ from urllib.error import URLError
++import sys
+
+ import pandas as pd
+ import pygal
+ import twitter
+ from authlib.common.errors import AuthlibBaseError
+-from authlib.integrations._client import MissingRequestTokenError
+ from flask import Flask, flash, g, redirect, render_template, request, session, url_for
+ from flask_migrate import Migrate
+ from authlib.integrations.flask_client import OAuth
+@@ -30,12 +30,17 @@ from moa.forms import MastodonIDForm, SettingsForm
+ from moa.helpers import blacklisted, email_bridge_details, send_blacklisted_email, timespan, FORMAT
+ from moa.models import Bridge, MastodonHost, TSettings, WorkerStat, metadata, BridgeStat, BridgeMetadata
+
++
++class MissingRequestTokenError(Exception):
++ pass
++
++
+ app = Flask(__name__)
+
+ formatter = logging.Formatter(FORMAT)
+
+ # initialize the log handler
+-logHandler = logging.FileHandler('logs/app.log')
++logHandler = logging.StreamHandler(sys.stderr)
+ logHandler.setFormatter(formatter)
+
+ # set the app logger level
+@@ -45,7 +50,6 @@ app.logger.addHandler(logHandler)
+
+ app.logger.info("Starting up...")
+
+-
+ config = os.environ.get('MOA_CONFIG', 'DevelopmentConfig')
+ # this is needed to get this in line with other modules, where we expect MOA_CONFIG to be an object in config.py.
+ app.config.from_object('config.' + config)
+@@ -59,14 +63,15 @@ if app.config['SENTRY_DSN']:
+ from sentry_sdk.integrations.flask import FlaskIntegration
+
+ sentry_logging = LoggingIntegration(
+- level=logging.INFO, # Capture info and above as breadcrumbs
+- event_level=logging.FATAL # Only send fatal errors as events
++ level=logging.INFO, # Capture info and above as breadcrumbs
++ event_level=logging.FATAL # Only send fatal errors as events
+ )
+
+- sentry_sdk.init(
+- dsn=app.config['SENTRY_DSN'],
+- integrations=[FlaskIntegration(), sentry_logging, SqlalchemyIntegration()]
+- )
++ sentry_sdk.init(dsn=app.config['SENTRY_DSN'],
++ integrations=[
++ FlaskIntegration(), sentry_logging,
++ SqlalchemyIntegration()
++ ])
+
+ db = SQLAlchemy(metadata=metadata)
+ migrate = Migrate(app, db)
+@@ -76,17 +81,17 @@ oauth = OAuth(app)
+
+ if app.config.get('TWITTER_CONSUMER_KEY', None):
+ oauth.register(
+- name='twitter',
+- client_id=app.config['TWITTER_CONSUMER_KEY'],
+- client_secret=app.config['TWITTER_CONSUMER_SECRET'],
+- request_token_url='https://api.twitter.com/oauth/request_token',
+- request_token_params=None,
+- access_token_url='https://api.twitter.com/oauth/access_token',
+- access_token_params=None,
+- authorize_url='https://api.twitter.com/oauth/authenticate',
+- authorize_params=None,
+- api_base_url='https://api.twitter.com/1.1/',
+- client_kwargs=None,
++ name='twitter',
++ client_id=app.config['TWITTER_CONSUMER_KEY'],
++ client_secret=app.config['TWITTER_CONSUMER_SECRET'],
++ request_token_url='https://api.twitter.com/oauth/request_token',
++ request_token_params=None,
++ access_token_url='https://api.twitter.com/oauth/access_token',
++ access_token_params=None,
++ authorize_url='https://api.twitter.com/oauth/authenticate',
++ authorize_params=None,
++ api_base_url='https://api.twitter.com/1.1/',
++ client_kwargs=None,
+ )
+
+ # mastodon_scopes = ["write:statuses", "write:media", "read:accounts", "read:statuses"]
+@@ -116,28 +121,32 @@ def index():
+ form = SettingsForm(obj=settings)
+
+ if 'bridge_id' in session:
+- bridge = db.session.query(Bridge).filter_by(id=session['bridge_id']).first()
++ bridge = db.session.query(Bridge).filter_by(
++ id=session['bridge_id']).first()
+
+ if bridge:
+ g.bridge = bridge
+ settings = bridge.t_settings
+- app.logger.debug(f"Existing settings found: {enabled} {settings.__dict__}")
++ app.logger.debug(
++ f"Existing settings found: {enabled} {settings.__dict__}")
+
+ form = SettingsForm(obj=settings)
+
+ if not bridge.mastodon_access_code or not bridge.twitter_oauth_token:
+ form.remove_masto_and_twitter_fields()
+
+- return render_template('index.html.j2',
+- form=form,
+- mform=mform,
+- )
++ return render_template(
++ 'index.html.j2',
++ form=form,
++ mform=mform,
++ )
+
+
+ @app.route('/options', methods=["POST"])
+ def options():
+ if 'bridge_id' in session:
+- bridge = db.session.query(Bridge).filter_by(id=session['bridge_id']).first()
++ bridge = db.session.query(Bridge).filter_by(
++ id=session['bridge_id']).first()
+ else:
+ flash('ERROR: Please log in to an account')
+ return redirect(url_for('index'))
+@@ -177,11 +186,11 @@ def catch_up_twitter(bridge):
+ if bridge.twitter_last_id == 0 and bridge.twitter_oauth_token:
+ # get twitter ID
+ twitter_api = twitter.Api(
+- consumer_key=app.config['TWITTER_CONSUMER_KEY'],
+- consumer_secret=app.config['TWITTER_CONSUMER_SECRET'],
+- access_token_key=bridge.twitter_oauth_token,
+- access_token_secret=bridge.twitter_oauth_secret,
+- tweet_mode='extended' # Allow tweets longer than 140 raw characters
++ consumer_key=app.config['TWITTER_CONSUMER_KEY'],
++ consumer_secret=app.config['TWITTER_CONSUMER_SECRET'],
++ access_token_key=bridge.twitter_oauth_token,
++ access_token_secret=bridge.twitter_oauth_secret,
++ tweet_mode='extended' # Allow tweets longer than 140 raw characters
+ )
+ try:
+ tl = twitter_api.GetUserTimeline()
+@@ -190,7 +199,8 @@ def catch_up_twitter(bridge):
+ else:
+ if len(tl) > 0:
+ bridge.twitter_last_id = tl[0].id
+- d = datetime.strptime(tl[0].created_at, '%a %b %d %H:%M:%S %z %Y')
++ d = datetime.strptime(tl[0].created_at,
++ '%a %b %d %H:%M:%S %z %Y')
+ bridge.md.last_tweet = d
+ else:
+ bridge.twitter_last_id = 0
+@@ -225,13 +235,15 @@ def catch_up_mastodon(bridge):
+ @app.route('/delete', methods=["POST"])
+ def delete():
+ if 'bridge_id' in session:
+- bridge = db.session.query(Bridge).filter_by(id=session['bridge_id']).first()
++ bridge = db.session.query(Bridge).filter_by(
++ id=session['bridge_id']).first()
+
+ if bridge:
+ app.logger.info(f"Deleting settings for Bridge {bridge.id}")
+ settings = bridge.t_settings
+ md = bridge.md
+- db.session.query(BridgeStat).filter_by(bridge_id=bridge.id).delete()
++ db.session.query(BridgeStat).filter_by(
++ bridge_id=bridge.id).delete()
+ db.session.delete(bridge)
+ db.session.delete(settings)
+ db.session.delete(md)
+@@ -246,11 +258,9 @@ def delete():
+
+ @app.route('/twitter_login')
+ def twitter_login():
+- callback_url = url_for(
+- 'twitter_oauthorized',
+- _external=True,
+- next=request.args.get('next')
+- )
++ callback_url = url_for('twitter_oauthorized',
++ _external=True,
++ next=request.args.get('next'))
+
+ app.logger.debug(callback_url)
+
+@@ -273,9 +283,12 @@ def twitter_oauthorized():
+ return redirect(url_for('index'))
+
+ if resp is None:
+- flash('ERROR: You denied the request to sign in or have cookies disabled.')
++ flash(
++ 'ERROR: You denied the request to sign in or have cookies disabled.'
++ )
+
+- elif blacklisted(resp['screen_name'], app.config.get('TWITTER_BLACKLIST', [])):
++ elif blacklisted(resp['screen_name'],
++ app.config.get('TWITTER_BLACKLIST', [])):
+ flash('ERROR: Access Denied.')
+ send_blacklisted_email(app, resp['screen_name'])
+
+@@ -306,18 +319,18 @@ def twitter_oauthorized():
+
+
+ def get_or_create_host(hostname):
+- mastodonhost = db.session.query(MastodonHost).filter_by(hostname=hostname).first()
++ mastodonhost = db.session.query(MastodonHost).filter_by(
++ hostname=hostname).first()
+
+ if not mastodonhost:
+
+ try:
+ client_id, client_secret = Mastodon.create_app(
+- "Moa",
+- scopes=mastodon_scopes,
+- api_base_url=f"https://{hostname}",
+- website="https://moa.party/",
+- redirect_uris=url_for("mastodon_oauthorized", _external=True)
+- )
++ "Moa",
++ scopes=mastodon_scopes,
++ api_base_url=f"https://{hostname}",
++ website="https://moa.party/",
++ redirect_uris=url_for("mastodon_oauthorized", _external=True))
+
+ app.logger.info(f"New host created for {hostname}")
+
+@@ -344,13 +357,11 @@ def mastodon_api(hostname, access_code=None):
+ mastodonhost = get_or_create_host(hostname)
+
+ if mastodonhost:
+- api = Mastodon(
+- client_id=mastodonhost.client_id,
+- client_secret=mastodonhost.client_secret,
+- api_base_url=f"https://{mastodonhost.hostname}",
+- access_token=access_code,
+- debug_requests=False
+- )
++ api = Mastodon(client_id=mastodonhost.client_id,
++ client_secret=mastodonhost.client_secret,
++ api_base_url=f"https://{mastodonhost.hostname}",
++ access_token=access_code,
++ debug_requests=False)
+
+ return api
+ return None
+@@ -408,12 +419,11 @@ def mastodon_login():
+
+ if api:
+ return redirect(
+- api.auth_request_url(
+- scopes=mastodon_scopes,
+- force_login=True,
+- redirect_uris=url_for("mastodon_oauthorized", _external=True)
+- )
+- )
++ api.auth_request_url(scopes=mastodon_scopes,
++ force_login=True,
++ redirect_uris=url_for(
++ "mastodon_oauthorized",
++ _external=True)))
+ else:
+ flash(f"There was a problem connecting to the mastodon server.")
+ else:
+@@ -435,7 +445,9 @@ def mastodon_oauthorized():
+ app.logger.info(f"Authorization code {authorization_code} for {host}")
+
+ if not host:
+- flash('There was an error. Please ensure you allow this site to use cookies.')
++ flash(
++ 'There was an error. Please ensure you allow this site to use cookies.'
++ )
+ return redirect(url_for('index'))
+
+ session.pop('mastodon_host', None)
+@@ -445,19 +457,23 @@ def mastodon_oauthorized():
+ local_scopes = mastodon_scopes
+
+ try:
+- access_code = api.log_in(
+- code=authorization_code,
+- scopes=local_scopes,
+- redirect_uri=url_for("mastodon_oauthorized", _external=True)
+- )
++ access_code = api.log_in(code=authorization_code,
++ scopes=local_scopes,
++ redirect_uri=url_for(
++ "mastodon_oauthorized",
++ _external=True))
+ except MastodonAPIError as e:
+ # Possibly a scopes problem?
+- flash(f"There was a problem connecting to the mastodon server. The error was {e}")
++ flash(
++ f"There was a problem connecting to the mastodon server. The error was {e}"
++ )
+ return redirect(url_for('index'))
+
+ except MastodonIllegalArgumentError as e:
+
+- flash(f"There was a problem connecting to the mastodon server. The error was {e}")
++ flash(
++ f"There was a problem connecting to the mastodon server. The error was {e}"
++ )
+ return redirect(url_for('index'))
+
+ # app.logger.info(f"Access code {access_code}")
+@@ -468,13 +484,17 @@ def mastodon_oauthorized():
+ creds = api.account_verify_credentials()
+
+ except (MastodonUnauthorizedError, MastodonAPIError) as e:
+- flash(f"There was a problem connecting to the mastodon server. The error was {e}")
++ flash(
++ f"There was a problem connecting to the mastodon server. The error was {e}"
++ )
+ return redirect(url_for('index'))
+
+ username = creds["username"]
+ account_id = creds["id"]
+
+- bridge = db.session.query(Bridge).filter_by(mastodon_account_id=account_id, mastodon_host_id=masto_host.id).first()
++ bridge = db.session.query(Bridge).filter_by(
++ mastodon_account_id=account_id,
++ mastodon_host_id=masto_host.id).first()
+
+ if bridge:
+ session['bridge_id'] = bridge.id
+@@ -520,7 +540,9 @@ def instagram_activate():
+ # app.logger.info(redirect_uri)
+
+ scope = ["basic"]
+- api = InstagramAPI(client_id=client_id, client_secret=client_secret, redirect_uri=redirect_uri)
++ api = InstagramAPI(client_id=client_id,
++ client_secret=client_secret,
++ redirect_uri=redirect_uri)
+
+ try:
+ redirect_uri = api.get_authorize_login_url(scope=scope)
+@@ -540,7 +562,9 @@ def instagram_oauthorized():
+ client_id = app.config['INSTAGRAM_CLIENT_ID']
+ client_secret = app.config['INSTAGRAM_SECRET']
+ redirect_uri = url_for('instagram_oauthorized', _external=True)
+- api = InstagramAPI(client_id=client_id, client_secret=client_secret, redirect_uri=redirect_uri)
++ api = InstagramAPI(client_id=client_id,
++ client_secret=client_secret,
++ redirect_uri=redirect_uri)
+
+ try:
+ access_token = api.exchange_code_for_access_token(code)
+@@ -565,15 +589,18 @@ def instagram_oauthorized():
+ bridge.instagram_account_id = data['id']
+ bridge.instagram_handle = data['username']
+
+- user_api = InstagramAPI(access_token=bridge.instagram_access_code, client_secret=client_secret)
++ user_api = InstagramAPI(access_token=bridge.instagram_access_code,
++ client_secret=client_secret)
+
+ try:
+- latest_media, _ = user_api.user_recent_media(user_id=bridge.instagram_account_id, count=1)
++ latest_media, _ = user_api.user_recent_media(
++ user_id=bridge.instagram_account_id, count=1)
+ except Exception:
+ latest_media = []
+
+ if len(latest_media) > 0:
+- bridge.instagram_last_id = datetime_to_timestamp(latest_media[0].created_time)
++ bridge.instagram_last_id = datetime_to_timestamp(
++ latest_media[0].created_time)
+ else:
+ bridge.instagram_last_id = 0
+
+@@ -595,8 +622,7 @@ def logout():
+ def stats():
+ hours = request.args.get('hours', 24)
+
+- return render_template('stats.html.j2',
+- hours=hours)
++ return render_template('stats.html.j2', hours=hours)
+
+
+ @app.route('/deactivate_account')
+@@ -634,9 +660,10 @@ def time_graph():
+ hours = int(request.args.get('hours', 24))
+
+ since = datetime.now() - timedelta(hours=hours)
+- stats_query = db.session.query(WorkerStat).filter(WorkerStat.created > since).with_entities(WorkerStat.created,
+- WorkerStat.time,
+- WorkerStat.worker)
++ stats_query = db.session.query(WorkerStat).filter(
++ WorkerStat.created > since).with_entities(WorkerStat.created,
++ WorkerStat.time,
++ WorkerStat.worker)
+
+ df = pd.read_sql(stats_query.statement, stats_query.session.bind)
+
+@@ -682,10 +709,11 @@ def count_graph():
+ hours = int(request.args.get('hours', 24))
+ since = datetime.now() - timedelta(hours=hours)
+
+- stats_query = db.session.query(WorkerStat).filter(WorkerStat.created > since).with_entities(WorkerStat.created,
+- WorkerStat.toots,
+- WorkerStat.tweets,
+- WorkerStat.instas)
++ stats_query = db.session.query(WorkerStat).filter(
++ WorkerStat.created > since).with_entities(WorkerStat.created,
++ WorkerStat.toots,
++ WorkerStat.tweets,
++ WorkerStat.instas)
+
+ df = pd.read_sql(stats_query.statement, stats_query.session.bind)
+ df.set_index(['created'], inplace=True)
+@@ -700,9 +728,10 @@ def count_graph():
+ tweets = r['tweets'].tolist()
+ instas = r['instas'].tolist()
+
+- chart = pygal.StackedBar(title=f"# of Incoming Messages ({timespan(hours)})\n{total} total",
+- human_readable=True,
+- legend_at_bottom=True)
++ chart = pygal.StackedBar(
++ title=f"# of Incoming Messages ({timespan(hours)})\n{total} total",
++ human_readable=True,
++ legend_at_bottom=True)
+ chart.add('Toots', toots)
+ chart.add('Tweets', tweets)
+ chart.add('Instas', instas)
+@@ -715,10 +744,11 @@ def percent_graph():
+ hours = int(request.args.get('hours', 24))
+ since = datetime.now() - timedelta(hours=hours)
+
+- stats_query = db.session.query(WorkerStat).filter(WorkerStat.created > since).with_entities(WorkerStat.created,
+- WorkerStat.toots,
+- WorkerStat.tweets,
+- WorkerStat.instas)
++ stats_query = db.session.query(WorkerStat).filter(
++ WorkerStat.created > since).with_entities(WorkerStat.created,
++ WorkerStat.toots,
++ WorkerStat.tweets,
++ WorkerStat.instas)
+
+ df = pd.read_sql(stats_query.statement, stats_query.session.bind)
+ df.set_index(['created'], inplace=True)
+@@ -736,9 +766,10 @@ def percent_graph():
+ tweets_p = r['tweets_p'].tolist()
+ instas_p = r['instas_p'].tolist()
+
+- chart = pygal.StackedBar(title=f"Ratio of Incoming Messages ({timespan(hours)})",
+- human_readable=True,
+- legend_at_bottom=True)
++ chart = pygal.StackedBar(
++ title=f"Ratio of Incoming Messages ({timespan(hours)})",
++ human_readable=True,
++ legend_at_bottom=True)
+ chart.add('Toots', toots_p)
+ chart.add('Tweets', tweets_p)
+ chart.add('Instas', instas_p)
+@@ -751,7 +782,8 @@ def user_graph():
+ hours = int(request.args.get('hours', 24))
+ since = datetime.now() - timedelta(hours=hours)
+
+- stats_query = db.session.query(Bridge).filter(Bridge.created > since).filter(Bridge.enabled == 1).with_entities(
++ stats_query = db.session.query(Bridge).filter(
++ Bridge.created > since).filter(Bridge.enabled == 1).with_entities(
+ Bridge.created)
+
+ base_count_query = db.session.query(func.count(Bridge.id)).scalar()
+@@ -793,4 +825,4 @@ def page_not_found(e):
+
+ if __name__ == '__main__':
+
+- app.run()
++ app.run(host="::1")
+diff --git a/flake.lock b/flake.lock
+new file mode 100644
+index 0000000..ddbf123
+--- /dev/null
++++ b/flake.lock
+@@ -0,0 +1,66 @@
++{
++ "nodes": {
++ "flake-utils": {
++ "locked": {
++ "lastModified": 1667395993,
++ "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
++ "owner": "numtide",
++ "repo": "flake-utils",
++ "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
++ "type": "github"
++ },
++ "original": {
++ "owner": "numtide",
++ "repo": "flake-utils",
++ "type": "github"
++ }
++ },
++ "nix-packages": {
++ "inputs": {
++ "flake-utils": [
++ "flake-utils"
++ ],
++ "nixpkgs": [
++ "nixpkgs"
++ ]
++ },
++ "locked": {
++ "lastModified": 1667423076,
++ "narHash": "sha256-0QUa2SBk/+S+q4hBpTSembUtHdiWjIex7tgb6jIoPkA=",
++ "ref": "refs/heads/main",
++ "rev": "31d36925f77febc11189dbc015f2593f17b9a4ff",
++ "revCount": 331,
++ "type": "git",
++ "url": "https://git.chir.rs/darkkirb/nix-packages"
++ },
++ "original": {
++ "type": "git",
++ "url": "https://git.chir.rs/darkkirb/nix-packages"
++ }
++ },
++ "nixpkgs": {
++ "locked": {
++ "lastModified": 1667456235,
++ "narHash": "sha256-bhRXd2WJt6uFTNPGVGJ0/KjssSmmz4kls/1Ggm3gZ0s=",
++ "owner": "NixOS",
++ "repo": "nixpkgs",
++ "rev": "60974988b3e1e5d61341a5ef0357e7ae99fcf7f1",
++ "type": "github"
++ },
++ "original": {
++ "owner": "NixOS",
++ "repo": "nixpkgs",
++ "type": "github"
++ }
++ },
++ "root": {
++ "inputs": {
++ "flake-utils": "flake-utils",
++ "nix-packages": "nix-packages",
++ "nixpkgs": "nixpkgs"
++ }
++ }
++ },
++ "root": "root",
++ "version": 7
++}
+diff --git a/flake.nix b/flake.nix
+new file mode 100644
+index 0000000..ff25756
+--- /dev/null
++++ b/flake.nix
+@@ -0,0 +1,53 @@
++{
++ inputs = {
++ nixpkgs.url = github:NixOS/nixpkgs;
++ flake-utils.url = github:numtide/flake-utils;
++ nix-packages.url = git+https://git.chir.rs/darkkirb/nix-packages;
++ nix-packages.inputs = {
++ nixpkgs.follows = "nixpkgs";
++ flake-utils.follows = "flake-utils";
++ };
++ };
++ outputs = {
++ self,
++ nixpkgs,
++ flake-utils,
++ nix-packages,
++ }:
++ flake-utils.lib.eachDefaultSystem (system: let
++ pkgs = import nixpkgs {inherit system;};
++ nix-pkgs = import nix-packages {inherit pkgs;};
++ in {
++ formatter = pkgs.alejandra;
++ devShells.default = pkgs.mkShell {
++ MOA_CONFIG = "DevelopmentConfig";
++ nativeBuildInputs = [
++ (pkgs.python3.withPackages (ps:
++ with ps; [
++ certifi
++ flask
++ flask_sqlalchemy
++ flask_mail
++ flask_migrate
++ flask_wtf
++ mastodon-py
++ pandas
++ psutil
++ pygal
++ python-twitter
++ pymysql
++ sentry-sdk
++ authlib
++ cairosvg
++ werkzeug
++ wheel
++ setuptools
++ nix-pkgs.python-instagram
++ psycopg2
++ ]))
++ pkgs.yapf
++ pkgs.sqlite
++ ];
++ };
++ });
++}
+diff --git a/migrations/alembic.ini b/migrations/alembic.ini
+index f8ed480..5301449 100644
+--- a/migrations/alembic.ini
++++ b/migrations/alembic.ini
+@@ -7,6 +7,7 @@
+ # set to 'true' to run the environment during
+ # the 'revision' command, regardless of autogenerate
+ # revision_environment = false
++script_location = .
+
+
+ # Logging configuration
+diff --git a/migrations/env.py b/migrations/env.py
+index 690d48b..c996c74 100755
+--- a/migrations/env.py
++++ b/migrations/env.py
+@@ -18,6 +18,7 @@ logger = logging.getLogger('alembic.env')
+ # from myapp import mymodel
+ # target_metadata = mymodel.Base.metadata
+ from flask import current_app
++
+ config.set_main_option('sqlalchemy.url',
+ current_app.config.get('SQLALCHEMY_DATABASE_URI'))
+ target_metadata = current_app.extensions['migrate'].db.metadata
+@@ -82,6 +83,7 @@ def run_migrations_online():
+ finally:
+ connection.close()
+
++
+ if context.is_offline_mode():
+ run_migrations_offline()
+ else:
+diff --git a/migrations/versions/b0780999e063_allow_deny_list.py b/migrations/versions/b0780999e063_allow_deny_list.py
+new file mode 100644
+index 0000000..5344b25
+--- /dev/null
++++ b/migrations/versions/b0780999e063_allow_deny_list.py
+@@ -0,0 +1,38 @@
++"""Allow/Deny list
++
++Revision ID: b0780999e063
++Revises: 3ac471544742
++Create Date: 2022-11-03 07:53:00
++
++"""
++from alembic import op
++import sqlalchemy as sa
++
++# revision identifiers, used by Alembic.
++revision = 'b0780999e063'
++down_revision = '3ac471544742'
++branch_labels = None
++depends_on = None
++
++
++def upgrade():
++ # ### commands auto generated by Alembic - please adjust! ###
++ op.create_table('block_allow_list',
++ sa.Column('id', sa.Integer(), nullable=False),
++ sa.Column('pattern', sa.String(length=100),
++ nullable=False),
++ sa.Column('allow', sa.Boolean(), nullable=False),
++ sa.Column('settings_id', sa.Integer(), nullable=False),
++ sa.PrimaryKeyConstraint('id'),
++ mysql_charset='utf8mb4',
++ mysql_collate='utf8mb4_general_ci')
++ op.create_foreign_key(None, 'block_allow_list', 'settings',
++ ['settings_id'], ['id'])
++
++ # ### end Alembic commands ###
++
++
++def downgrade():
++ # ### commands auto generated by Alembic - please adjust! ###
++ op.drop_table('block_allow_list')
++ # ### end Alembic commands ###
+diff --git a/moa/models.py b/moa/models.py
+index e57604b..050e87c 100644
+--- a/moa/models.py
++++ b/moa/models.py
+@@ -51,22 +51,33 @@ CON_XP_ONLYIF_TAGS = ['moa', 'xp']
+ CON_XP_UNLESS = 'unless'
+ CON_XP_UNLESS_TAGS = ['nomoa', 'noxp']
+
++
+ class TSettings(Base):
+ __tablename__ = 'settings'
+- __table_args__ = {'mysql_charset': 'utf8mb4', 'mysql_collate': 'utf8mb4_general_ci'}
++ __table_args__ = {
++ 'mysql_charset': 'utf8mb4',
++ 'mysql_collate': 'utf8mb4_general_ci'
++ }
+
+ id = Column(Integer, primary_key=True)
+ bridge = relationship('Bridge', backref='t_settings', lazy='dynamic')
+- conditional_posting = Column(String(10), nullable=False, server_default=CON_XP_DISABLED, default=CON_XP_DISABLED)
++ allow_deny_list = relationship('AllowDenyList', lazy='dynamic')
++ conditional_posting = Column(String(10),
++ nullable=False,
++ server_default=CON_XP_DISABLED,
++ default=CON_XP_DISABLED)
+
+ # Masto -> Twitter
+- post_to_twitter = Column(Boolean, nullable=False, default=True) # This means post public toots
++ post_to_twitter = Column(Boolean, nullable=False,
++ default=True) # This means post public toots
+ post_private_to_twitter = Column(Boolean, nullable=False, default=False)
+ post_unlisted_to_twitter = Column(Boolean, nullable=False, default=False)
+ split_twitter_messages = Column(Boolean, nullable=False, default=True)
+ post_boosts_to_twitter = Column(Boolean, nullable=False, default=True)
+ post_sensitive_behind_link = Column(Boolean, nullable=False, default=False)
+- sensitive_link_text = Column(String(100), nullable=False, default='(NSFW Image)')
++ sensitive_link_text = Column(String(100),
++ nullable=False,
++ default='(NSFW Image)')
+ remove_cw = Column(Boolean, nullable=False, default=False)
+
+ # Twitter -> Masto
+@@ -75,7 +86,9 @@ class TSettings(Base):
+ post_quotes_to_mastodon = Column(Boolean, nullable=False, default=True)
+ toot_visibility = Column(String(40), nullable=False, default='public')
+ tweets_behind_cw = Column(Boolean, nullable=False, default=False)
+- tweet_cw_text = Column(String(100), nullable=False, default="From birdsite")
++ tweet_cw_text = Column(String(100),
++ nullable=False,
++ default="From birdsite")
+
+ instagram_post_to_twitter = Column(Boolean, nullable=False, default=False)
+ instagram_post_to_mastodon = Column(Boolean, nullable=False, default=False)
+@@ -116,6 +129,19 @@ class TSettings(Base):
+ self.post_rts_to_mastodon
+
+
++class AllowDenyList(Base):
++ __tablename__ = "block_allow_list"
++ __table_args__ = {
++ 'mysql_charset': 'utf8mb4',
++ 'mysql_collate': 'utf8mb4_general_ci'
++ }
++
++ id = Column(Integer, primary_key=True)
++ pattern = Column(String(100), nullable=False)
++ allow = Column(Boolean, nullable=False)
++ settings_id = Column(Integer, ForeignKey('settings.id'), nullable=False)
++
++
+ class Bridge(Base):
+ __tablename__ = 'bridge'
+
+@@ -139,7 +165,9 @@ class Bridge(Base):
+ instagram_handle = Column(String(30))
+
+ t_settings_id = Column(Integer, ForeignKey('settings.id'), nullable=True)
+- metadata_id = Column(Integer, ForeignKey('bridgemetadata.id'), nullable=True)
++ metadata_id = Column(Integer,
++ ForeignKey('bridgemetadata.id'),
++ nullable=True)
+
+ created = Column(DateTime, default=datetime.utcnow)
+ updated = Column(DateTime)
+@@ -258,7 +286,7 @@ if __name__ == '__main__':
+ import pymysql
+
+ engine = create_engine(config.SQLALCHEMY_DATABASE_URI)
+- metadata = MetaData(engine, reflect=True)
++ metadata = MetaData(engine)
+ print("Creating Tables")
+
+ Base.metadata.create_all(engine)
+diff --git a/moa/toot.py b/moa/toot.py
+index 08aa61d..b6dc5a3 100644
+--- a/moa/toot.py
++++ b/moa/toot.py
+@@ -8,11 +8,11 @@ from moa.message import Message
+ from moa.models import CON_XP_ONLYIF, CON_XP_ONLYIF_TAGS, CON_XP_UNLESS, CON_XP_UNLESS_TAGS
+ from moa.tweet import HOUR_CUTOFF
+
+-MY_TLDS = [
+- "shop"
+-]
++MY_TLDS = ["shop"]
+
+-URL_REGEXP = re.compile(r"([--:\w?@%&+~#=]*\.[a-z]{2,4}\/{0,2})((?:[?&](?:\w+)=(?:\w+))+|[--:\w?@%&+~#=]+)?", re.U | re.I)
++URL_REGEXP = re.compile(
++ r"([--:\w?@%&+~#=]*\.[a-z]{2,4}\/{0,2})((?:[?&](?:\w+)=(?:\w+))+|[--:\w?@%&+~#=]+)?",
++ re.U | re.I)
+
+ logger = logging.getLogger('worker')
+
+@@ -69,7 +69,8 @@ class Toot(Message):
+
+ @property
+ def is_self_reply(self):
+- return self.is_reply and self.data['in_reply_to_account_id'] == self.data['account']['id']
++ return self.is_reply and self.data[
++ 'in_reply_to_account_id'] == self.data['account']['id']
+
+ @property
+ def is_boost(self):
+@@ -111,7 +112,6 @@ class Toot(Message):
+
+ @property
+ def should_skip(self):
+-
+ if self.too_old:
+ logger.info(f'Skipping because >= {HOUR_CUTOFF} hours old.')
+ return True
+@@ -125,6 +125,30 @@ class Toot(Message):
+ logger.info(f'Skipping reply.')
+ return True
+
++ found = True
++ for entry in self.settings.allow_deny_list:
++ hasEntry = True
++ toot_tags = {x.name for x in self.data['tags']}
++ if entry.allow:
++ if entry.pattern not in self.content:
++ found = False
++ break
++ if entry.pattern.startswith(
++ "#") and entry.pattern[1:] not in toot_tags:
++ found = False
++ break
++ else:
++ if entry.pattern in self.content:
++ found = False
++ break
++ if entry.pattern.startswith(
++ "#") and entry.pattern[1:] in toot_tags:
++ found = False
++ break
++
++ if not found:
++ return True
++
+ if self.visibility == 'private' and not self.settings.post_private_to_twitter:
+ logger.info(f'Skipping: Not Posting Private toots.')
+ return True
+@@ -202,12 +226,14 @@ class Toot(Message):
+ status_length += self.url_length
+
+ if self.is_sensitive and self.settings.post_sensitive_behind_link:
+- status_length += len(f"\n{self.settings.sensitive_link_text}\n{self.url}")
++ status_length += len(
++ f"\n{self.settings.sensitive_link_text}\n{self.url}")
+
+ return status_length
+
+ def sanitize_twitter_handles(self):
+- self.content = re.sub(r'@?(\w{1,15})@twitter.com', '\g<1>', self.content)
++ self.content = re.sub(r'@?(\w{1,15})@twitter.com', '\g<1>',
++ self.content)
+
+ # find possible twitter handles so we can get their ranges
+ tm = list(re.finditer(r'@(\w{1,15})', self.content))
+@@ -250,43 +276,51 @@ class Toot(Message):
+ @property
+ def clean_content(self):
+
+- media_regexp = re.compile(re.escape(self.instance_url) + "\/media\/[\w-]+\s?")
++ media_regexp = re.compile(
++ re.escape(self.instance_url) + "\/media\/[\w-]+\s?")
+
+ if not self.content:
+
+ self.content = self.raw_content
+
+ # We trust mastodon to return valid HTML
+- self.content = re.sub(r']*href="([^"]+)">[^<]*', '\g<1>', self.content)
++ self.content = re.sub(r']*href="([^"]+)">[^<]*', '\g<1>',
++ self.content)
+
+ # We replace html br with new lines
+- self.content = "\n".join(re.compile(r'
', re.IGNORECASE).split(self.content))
++ self.content = "\n".join(
++ re.compile(r'
', re.IGNORECASE).split(self.content))
+
+ # We must also replace new paragraphs with double line skips
+- self.content = "\n\n".join(re.compile(r'
', re.IGNORECASE).split(self.content))
++ self.content = "\n\n".join(
++ re.compile(r'
', re.IGNORECASE).split(self.content))
+
+ # Then we can delete the other html contents and unescape the string
+- self.content = html.unescape(str(re.compile(r'<.*?>').sub("", self.content).strip()))
++ self.content = html.unescape(
++ str(re.compile(r'<.*?>').sub("", self.content).strip()))
+
+ # Trim out media URLs
+ self.content = re.sub(media_regexp, "", self.content)
+
+ # fix up masto mentions
+ for mention in self.mentions:
+- self.content = re.sub(f'@({mention[0]})(?!@)', f"{mention[1]}", self.content)
++ self.content = re.sub(f'@({mention[0]})(?!@)', f"{mention[1]}",
++ self.content)
+
+ if self.config.SANITIZE_TWITTER_HANDLES:
+ self.sanitize_twitter_handles()
+
+ else:
+- self.content = re.sub(r'@(\w{1,15})@twitter.com', '@\g<1>', self.content)
++ self.content = re.sub(r'@(\w{1,15})@twitter.com', '@\g<1>',
++ self.content)
+
+ self.content = self.content.strip()
+
+ if self.spoiler_text and not self.settings.remove_cw:
+ self.content = f"CW: {self.spoiler_text}\n\n{self.content}"
+
+- if self.is_sensitive and self.settings.post_sensitive_behind_link and len(self.media_attachments) > 0:
++ if self.is_sensitive and self.settings.post_sensitive_behind_link and len(
++ self.media_attachments) > 0:
+ self.content = f"{self.content}\n{self.settings.sensitive_link_text}\n{self.url}"
+
+ if self.is_boost:
+@@ -318,7 +352,9 @@ class Toot(Message):
+ words = self.clean_content.split(" ")
+
+ if self.settings.split_twitter_messages:
+- logger.info(f'Toot bigger than {max_length} characters, need to split...')
++ logger.info(
++ f'Toot bigger than {max_length} characters, need to split...'
++ )
+
+ for next_word in words:
+
+@@ -348,7 +384,8 @@ class Toot(Message):
+ self.message_parts.append(current_part.strip())
+
+ for i, msg in enumerate(self.message_parts):
+- self.message_parts[i] = msg.replace('XXXXX', f"({i+1}/{len(self.message_parts)})")
++ self.message_parts[i] = msg.replace(
++ 'XXXXX', f"({i+1}/{len(self.message_parts)})")
+ logger.debug(self.message_parts[i])
+ else:
+ logger.info('Truncating toot')
+diff --git a/moa/worker.py b/moa/worker.py
+index 733cb48..9c89934 100644
+--- a/moa/worker.py
++++ b/moa/worker.py
+@@ -45,15 +45,22 @@ if c.SENTRY_DSN:
+ from sentry_sdk.integrations.logging import LoggingIntegration
+
+ sentry_logging = LoggingIntegration(
+- level=logging.INFO, # Capture info and above as breadcrumbs
+- event_level=logging.FATAL # Only send fatal errors as events
++ level=logging.INFO, # Capture info and above as breadcrumbs
++ event_level=logging.FATAL # Only send fatal errors as events
+ )
+- sentry_sdk.init(dsn=c.SENTRY_DSN, integrations=[sentry_logging,
+- SqlalchemyIntegration(),
+- FlaskIntegration()])
++ sentry_sdk.init(dsn=c.SENTRY_DSN,
++ integrations=[
++ sentry_logging,
++ SqlalchemyIntegration(),
++ FlaskIntegration()
++ ])
+
+ parser = argparse.ArgumentParser(description='Moa Worker')
+-parser.add_argument('--worker', dest='worker', type=int, required=False, default=1)
++parser.add_argument('--worker',
++ dest='worker',
++ type=int,
++ required=False,
++ default=1)
+ args = parser.parse_args()
+
+ worker_stat = WorkerStat(worker=args.worker)
+@@ -87,7 +94,7 @@ except exc.SQLAlchemyError as e:
+
+ session = Session(engine)
+
+-lockfile = Path(f'worker_{args.worker}.lock')
++lockfile = Path(f'/tmp/moa_worker_{args.worker}.lock')
+
+
+ def check_worker_stop():
+@@ -131,11 +138,13 @@ with lockfile.open('wt') as f:
+ if not c.SEND:
+ l.warning("SENDING IS NOT ENABLED")
+
+-bridges = session.query(Bridge).filter_by(enabled=True).filter(BridgeMetadata.worker_id == args.worker).filter(BridgeMetadata.id == Bridge.metadata_id)
++bridges = session.query(Bridge).filter_by(enabled=True).filter(
++ BridgeMetadata.worker_id == args.worker).filter(
++ BridgeMetadata.id == Bridge.metadata_id)
+
+ l.info(f"Working on {bridges.count()} bridges")
+
+-if 'sqlite' not in c.SQLALCHEMY_DATABASE_URI and not c.DEVELOPMENT:
++if 'sqlite' not in c.SQLALCHEMY_DATABASE_URI and 'postgresql' not in c.SQLALCHEMY_DATABASE_URI and not c.DEVELOPMENT:
+ bridges = bridges.order_by(func.rand())
+
+ bridge_count = 0
+@@ -173,32 +182,38 @@ for bridge in bridges:
+ mastodon_last_id = bridge.mastodon_last_id
+ mastodonhost = bridge.mastodon_host
+
+- if mastodonhost.defer_until and mastodonhost.defer_until > datetime.now():
++ if mastodonhost.defer_until and mastodonhost.defer_until > datetime.now(
++ ):
+ l.warning(f"Deferring connections to {mastodonhost.hostname}")
+ continue
+
+- mast_api = Mastodon(
+- client_id=mastodonhost.client_id,
+- client_secret=mastodonhost.client_secret,
+- api_base_url=f"https://{mastodonhost.hostname}",
+- access_token=bridge.mastodon_access_code,
+- debug_requests=False,
+- request_timeout=15,
+- ratelimit_method='throw'
+- )
++ mast_api = Mastodon(client_id=mastodonhost.client_id,
++ client_secret=mastodonhost.client_secret,
++ api_base_url=f"https://{mastodonhost.hostname}",
++ access_token=bridge.mastodon_access_code,
++ debug_requests=False,
++ request_timeout=15,
++ ratelimit_method='throw')
+
+ try:
+ new_toots = mast_api.account_statuses(
+- bridge.mastodon_account_id,
+- since_id=bridge.mastodon_last_id
+- )
++ bridge.mastodon_account_id, since_id=bridge.mastodon_last_id)
+ except (MastodonAPIError, MastodonNetworkError) as e:
+ msg = f"{bridge.mastodon_user}@{mastodonhost.hostname} MastodonAPIError: {e}"
+ l.error(msg)
+
+- if any(x in repr(e) for x in ['revoked', 'invalid', 'not found', 'Forbidden', 'Unauthorized', 'Bad Request',
+- 'Name or service not known',]):
+- l.warning(f"Disabling bridge for user {bridge.mastodon_user}@{mastodonhost.hostname}")
++ if any(x in repr(e) for x in [
++ 'revoked',
++ 'invalid',
++ 'not found',
++ 'Forbidden',
++ 'Unauthorized',
++ 'Bad Request',
++ 'Name or service not known',
++ ]):
++ l.warning(
++ f"Disabling bridge for user {bridge.mastodon_user}@{mastodonhost.hostname}"
++ )
+ bridge.mastodon_access_code = None
+ bridge.enabled = False
+ else:
+@@ -273,9 +288,11 @@ for bridge in bridges:
+ except MastodonRatelimitError as e:
+ l.error(f"{bridge.mastodon_user}@{mastodonhost.hostname}: {e}")
+
+- if len(new_toots) > c.MAX_MESSAGES_PER_RUN:
+- l.error(f"{bridge.mastodon_user}@{mastodonhost.hostname}: Limiting to {c.MAX_MESSAGES_PER_RUN} messages")
+- new_toots = new_toots[-c.MAX_MESSAGES_PER_RUN:]
++ #if len(new_toots) > c.MAX_MESSAGES_PER_RUN:
++ # l.error(
++ # f"{bridge.mastodon_user}@{mastodonhost.hostname}: Limiting to {c.MAX_MESSAGES_PER_RUN} messages"
++ # )
++ # new_toots = new_toots[-c.MAX_MESSAGES_PER_RUN:]
+
+ if c.SEND and len(new_toots) != 0:
+ try:
+@@ -296,18 +313,18 @@ for bridge in bridges:
+ twitter_last_id = bridge.twitter_last_id
+
+ twitter_api = twitter.Api(
+- consumer_key=c.TWITTER_CONSUMER_KEY,
+- consumer_secret=c.TWITTER_CONSUMER_SECRET,
+- access_token_key=bridge.twitter_oauth_token,
+- access_token_secret=bridge.twitter_oauth_secret,
+- tweet_mode='extended' # Allow tweets longer than 140 raw characters
++ consumer_key=c.TWITTER_CONSUMER_KEY,
++ consumer_secret=c.TWITTER_CONSUMER_SECRET,
++ access_token_key=bridge.twitter_oauth_token,
++ access_token_secret=bridge.twitter_oauth_secret,
++ tweet_mode='extended' # Allow tweets longer than 140 raw characters
+ )
+
+ try:
+ new_tweets = twitter_api.GetUserTimeline(
+- since_id=bridge.twitter_last_id,
+- include_rts=True,
+- exclude_replies=False)
++ since_id=bridge.twitter_last_id,
++ include_rts=True,
++ exclude_replies=False)
+
+ except TwitterError as e:
+ l.error(f"@{bridge.twitter_handle}: {e}")
+@@ -320,7 +337,9 @@ for bridge in bridges:
+
+ elif isinstance(e.message, list) and len(e.message) > 0:
+ if e.message[0]['code'] in [89, 326]:
+- l.warning(f"Disabling bridge for Twitter user {bridge.twitter_handle}")
++ l.warning(
++ f"Disabling bridge for Twitter user {bridge.twitter_handle}"
++ )
+ bridge.twitter_oauth_token = None
+ bridge.twitter_oauth_secret = None
+ bridge.enabled = False
+@@ -331,7 +350,9 @@ for bridge in bridges:
+ continue
+
+ if len(new_tweets) > c.MAX_MESSAGES_PER_RUN:
+- l.error(f"@{bridge.twitter_handle}: Limiting to {c.MAX_MESSAGES_PER_RUN} messages")
++ l.error(
++ f"@{bridge.twitter_handle}: Limiting to {c.MAX_MESSAGES_PER_RUN} messages"
++ )
+ new_tweets = new_tweets[-c.MAX_MESSAGES_PER_RUN:]
+
+ if c.SEND and len(new_tweets) != 0:
+@@ -399,7 +420,9 @@ for bridge in bridges:
+ tweet_poster = TweetPoster(c.SEND, session, twitter_api, bridge)
+
+ if bridge.mastodon_access_code:
+- l.info(f"{bridge.id}: M - {bridge.mastodon_user}@{mastodonhost.hostname}")
++ l.info(
++ f"{bridge.id}: M - {bridge.mastodon_user}@{mastodonhost.hostname}"
++ )
+
+ tweet_poster = TweetPoster(c.SEND, session, twitter_api, bridge)
+
+@@ -458,7 +481,6 @@ for bridge in bridges:
+ bridge.md.last_tweet = tweet.created_at
+ session.commit()
+
+-
+ #
+ # Post Instagram
+ #
+@@ -489,7 +511,8 @@ for bridge in bridges:
+ stat_recorded = True
+
+ if not insta.should_skip_twitter and bridge.twitter_oauth_token:
+- tweet_poster = TweetPoster(c.SEND, session, twitter_api, bridge)
++ tweet_poster = TweetPoster(c.SEND, session, twitter_api,
++ bridge)
+
+ try:
+ result = tweet_poster.post(insta)
+@@ -524,7 +547,9 @@ if len(c.HEALTHCHECKS) >= args.worker:
+ except Exception:
+ pass
+
+-l.info(f"-- All done -> Total time: {worker_stat.formatted_time} / {worker_stat.items} items / {bridge_count} Bridges")
++l.info(
++ f"-- All done -> Total time: {worker_stat.formatted_time} / {worker_stat.items} items / {bridge_count} Bridges"
++)
+
+ session.add(worker_stat)
+