From 697fc7ce0e072ee3370006e4f2cbcf6c7bc48b72 Mon Sep 17 00:00:00 2001 From: Facundo Olano Date: Fri, 24 May 2024 16:34:35 -0300 Subject: [PATCH] Remove backlog and unwrap features (#96) * remove pop backlog task * remove backlog from tasks and routes * remove entry unwrap * remove backlogged from queries * remove backlogged from db * remove test * remove backlog and unwrap from templates --- feedi/models.py | 37 +----------- feedi/routes.py | 56 ------------------- feedi/scraping.py | 7 --- feedi/tasks.py | 20 ------- feedi/templates/base.html | 10 ---- feedi/templates/entry_commands.html | 5 -- feedi/templates/entry_header.html | 13 ----- feedi/templates/entry_list_page.html | 13 ----- .../f11ef7901ceb_remove_backlogged.py | 36 ++++++++++++ tests/test_routes.py | 44 --------------- 10 files changed, 37 insertions(+), 204 deletions(-) create mode 100644 migrations/versions/f11ef7901ceb_remove_backlogged.py diff --git a/feedi/models.py b/feedi/models.py index 8fd1912..3f9ea0a 100644 --- a/feedi/models.py +++ b/feedi/models.py @@ -511,7 +511,6 @@ class Entry(db.Model): viewed = sa.Column(sa.TIMESTAMP, index=True) favorited = sa.Column(sa.TIMESTAMP, index=True) pinned = sa.Column(sa.TIMESTAMP, index=True) - backlogged = sa.Column(sa.TIMESTAMP, index=True) raw_data = sa.orm.deferred(sa.Column(sa.String, doc="The original entry data received from the feed, as JSON")) @@ -572,32 +571,8 @@ def fetch_content(self): except Exception as e: logger.debug("failed to fetch content %s", e) - def embedded_links(self): - "Return a list of (url, title) for links found in the content_short (excluding hashtags and mentions)." - if self.content_short: - return [url for (url, text) in scraping.extract_links(self.target_url, self.content_short) - # skip hashtag and profile links - if not text.startswith('#') and not text.startswith('@')] - return [] - - def is_unwrappable(self): - "Return whether there are embedded links that can be extracted from the content_shrot." - return bool(self.embedded_links()) - - def backlog(self): - "Put this entry in the backlog." - self.backlogged = datetime.datetime.utcnow() - self.pinned = None - - def unbacklog(self): - "Pop this entry from the backlog." - self.backlogged = None - self.viewed = None - self.sort_date = datetime.datetime.utcnow() - self.fetch_content() - @classmethod - def _filtered_query(cls, user_id, hide_seen=False, favorited=None, backlogged=None, + def _filtered_query(cls, user_id, hide_seen=False, favorited=None, feed_name=None, username=None, folder=None, older_than=None, text=None): """ @@ -619,14 +594,6 @@ def _filtered_query(cls, user_id, hide_seen=False, favorited=None, backlogged=No if favorited: query = query.filter(cls.favorited.is_not(None)) - if backlogged: - query = query.filter(cls.backlogged.is_not(None)) - elif hide_seen: - # exclude backlogged unless explicitly asking for backlog list - # or if the hide seen option is disabled. - # (abusing the fact that non mixed lists set this flag to false) - query = query.filter(cls.backlogged.is_(None)) - if feed_name: query = query.filter(cls.feed.has(name=feed_name)) @@ -664,8 +631,6 @@ def sorted_by(cls, user_id, ordering, start_at, **filters): if filters.get('favorited'): return query.order_by(cls.favorited.desc()) - if filters.get('backlogged'): - return query.order_by(cls.backlogged) elif ordering == cls.ORDER_RECENCY: # reverse chronological order diff --git a/feedi/routes.py b/feedi/routes.py index b061e43..6d39800 100644 --- a/feedi/routes.py +++ b/feedi/routes.py @@ -16,7 +16,6 @@ @app.route("/users/") @app.route("/favorites", defaults={'favorited': True}, endpoint='favorites') -@app.route("/backlog", defaults={'backlogged': True}, endpoint='backlog') @app.route("/folder/") @app.route("/feeds//entries") @app.route("/") @@ -161,7 +160,6 @@ def autocomplete(): static_options = [ ('Home', flask.url_for('entry_list'), 'fas fa-home'), ('Favorites', flask.url_for('favorites', favorited=True), 'far fa-star'), - ('Backlog', flask.url_for('backlog', favorited=True), 'fa fa-archive'), ('Add Feed', flask.url_for('feed_add'), 'fas fa-plus'), ('Manage Feeds', flask.url_for('feed_list'), 'fas fa-edit'), ('Mastodon login', flask.url_for('mastodon_oauth'), 'fab fa-mastodon'), @@ -190,7 +188,6 @@ def entry_pin(id): else: entry.fetch_content() entry.pinned = datetime.datetime.utcnow() - entry.backlogged = None db.session.commit() # get the new list of pinned based on filters @@ -220,34 +217,6 @@ def entry_favorite(id): return '', 204 -@app.put("/backlog/") -@login_required -def entry_backlog_push(id): - "Put the entry of the given id in the backlog." - entry = db.get_or_404(models.Entry, id) - if entry.user_id != current_user.id: - flask.abort(404) - - entry.backlog() - db.session.commit() - return '', 204 - - -@app.delete("/backlog/") -@login_required -def entry_backlog_pop(id): - "Remove the entry of the given id from the backlog, sending it back to the home feed." - entry = db.get_or_404(models.Entry, id) - if entry.user_id != current_user.id: - flask.abort(404) - - if entry.backlogged: - entry.unbacklog() - - db.session.commit() - return '', 204 - - @app.put("/mastodon/favorites/") @login_required def mastodon_favorite(id): @@ -421,7 +390,6 @@ def feed_delete(feed_name): update = db.update(models.Entry)\ .where((models.Entry.feed_id == feed.id) & ( models.Entry.favorited.isnot(None) | - models.Entry.backlogged.isnot(None) | models.Entry.pinned.isnot(None)))\ .values(feed_id=None) db.session.execute(update) @@ -477,30 +445,6 @@ def entry_add(): return '', 204 -@app.post("/entries/") -@login_required -def entry_unwrap(id): - "If the entry has embedded links in its short content, extract the first and render it." - entry = db.get_or_404(models.Entry, id) - if entry.user_id != current_user.id: - flask.abort(404) - - if entry.content_short: - # If there's an inline link, "unwrap it", e.g. remove the old entry and put the linked - # article entry in its place - for link in entry.embedded_links(): - try: - subentry = models.Entry.from_url(current_user.id, link) - entry.viewed = datetime.datetime.now() - db.session.add(subentry) - db.session.commit() - return flask.render_template('entry_list_page.html', - entries=[subentry]) - except Exception: - continue - return "Couldn't unwrap", 400 - - @app.get("/entries/") @login_required def entry_view(id): diff --git a/feedi/scraping.py b/feedi/scraping.py index 7770f86..0708c0c 100644 --- a/feedi/scraping.py +++ b/feedi/scraping.py @@ -86,13 +86,6 @@ def all_meta(soup): return result -def extract_links(url, html): - soup = BeautifulSoup(html, 'lxml') - # checks tag.text so it skips image links - links = soup.find_all(lambda tag: tag.name == 'a' and tag.text and 'href' in tag) - return [(make_absolute(url, a['href']), a.text) for a in links] - - def make_absolute(url, path): "If `path` is a relative url, join it with the given absolute url." if not urllib.parse.urlparse(path).netloc: diff --git a/feedi/tasks.py b/feedi/tasks.py index f986571..b1badb2 100644 --- a/feedi/tasks.py +++ b/feedi/tasks.py @@ -121,7 +121,6 @@ def delete_old_entries(): .join(models.Feed.entries)\ .filter(models.Entry.sort_date < older_than_date, models.Entry.favorited.is_(None), - models.Entry.backlogged.is_(None), models.Entry.pinned.is_(None) )\ .group_by(models.Feed.id)\ @@ -144,7 +143,6 @@ def delete_old_entries(): q = db.delete(models.Entry)\ .where( models.Entry.favorited.is_(None), - models.Entry.backlogged.is_(None), models.Entry.pinned.is_(None), models.Entry.feed_id == feed_id, models.Entry.sort_date < min_sort_date, @@ -169,24 +167,6 @@ def delete_old_entries(): app.logger.info("Deleted %s old standalone entries from", res.rowcount) -@feed_cli.command('backlog') -@huey_task(crontab(minute='0', hour='0')) -def pop_backlog(): - "Periodically pop an entry from the backlog into the home feed." - # TODO make this configurable - week_ago = datetime.datetime.utcnow() - datetime.timedelta(days=7) - backlogged_date = sa.func.min(models.Entry.backlogged).label('backlogged_date') - query = db.select(models.Entry)\ - .group_by(models.Entry.user_id)\ - .having(backlogged_date < week_ago) - - for entry in db.session.scalars(query): - entry.unbacklog() - app.logger.info("Popped from user %s backlog: %s ", entry.user_id, entry.target_url) - - db.session.commit() - - @feed_cli.command('debug') @click.argument('url') def debug_feed(url): diff --git a/feedi/templates/base.html b/feedi/templates/base.html index c3fb255..4295f79 100644 --- a/feedi/templates/base.html +++ b/feedi/templates/base.html @@ -24,8 +24,6 @@ {% endfor %} favorites - - backlog
{% endif %} {{ self.sidebar_right() }} @@ -86,14 +84,6 @@

feedi

-
  • -
    -
    - - backlog -
    -
    -
    • {% endblock %} diff --git a/feedi/templates/entry_commands.html b/feedi/templates/entry_commands.html index 8e0c7aa..abd4dac 100644 --- a/feedi/templates/entry_commands.html +++ b/feedi/templates/entry_commands.html @@ -12,11 +12,6 @@ {% if entry.content_url and entry.id and request.path != url_for('entry_view', id=entry.id) %} View in reader {% endif %} -{% if not entry.backlogged %} - Send to backlog -{% endif %}
    Copy URL diff --git a/feedi/templates/entry_header.html b/feedi/templates/entry_header.html index 62e3b4c..5c54e7b 100644 --- a/feedi/templates/entry_header.html +++ b/feedi/templates/entry_header.html @@ -83,19 +83,6 @@ href="{{ entry.comments_url}}" target="_blank" > {% endif %} - {% if entry.backlogged %} - - {% elif entry.is_unwrappable() %} - - {% endif %}
    diff --git a/feedi/templates/entry_list_page.html b/feedi/templates/entry_list_page.html index 4d63d85..dae77a8 100644 --- a/feedi/templates/entry_list_page.html +++ b/feedi/templates/entry_list_page.html @@ -83,19 +83,6 @@ {% endif %} - {% if entry.backlogged %} - - {% elif entry.is_unwrappable() %} - - {% endif %} None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('entries', schema=None) as batch_op: + batch_op.drop_index('ix_entries_backlogged') + batch_op.drop_column('backlogged') + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('entries', schema=None) as batch_op: + batch_op.add_column(sa.Column('backlogged', sa.TIMESTAMP(), nullable=True)) + batch_op.create_index('ix_entries_backlogged', ['backlogged'], unique=False) + + # ### end Alembic commands ### diff --git a/tests/test_routes.py b/tests/test_routes.py index 31b8b61..e52fff2 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -202,50 +202,6 @@ def test_favorites(client): assert response.text.find('my-second-article') < response.text.find('my-third-article') -def test_backlog(client): - # add feed with a couple of articles - feed_domain = 'feed1.com' - response = create_feed(client, feed_domain, [{'title': 'my-third-article', 'date': '2023-11-10 00:00Z'}, - {'title': 'my-second-article', - 'date': '2023-10-10 00:00Z'}, - {'title': 'my-first-article', 'date': '2023-10-01 00:00Z'}]) - entry_ids = extract_entry_ids(response) - - # backlog the 3rd, then the 2nd - response = client.put(f'/backlog/{entry_ids[0]}') - assert response.status_code == 204 - response = client.put(f'/backlog/{entry_ids[1]}') - assert response.status_code == 204 - - # verify backlogged is excluded from home list - response = client.get('/backlog') - assert 'my-first-article' not in response.text - assert 'my-second-article' in response.text - assert 'my-third-article' in response.text - - # verify backlogged is included in the backlog, in the order they were added - response = client.get('/') - assert 'my-first-article' in response.text - assert 'my-second-article' not in response.text - assert 'my-third-article' not in response.text - - # pop from backlog - response = client.delete(f'/backlog/{entry_ids[0]}') - assert response.status_code == 204 - - # verify backlogged is included in home list - response = client.get('/backlog') - assert 'my-first-article' not in response.text - assert 'my-second-article' in response.text - assert 'my-third-article' not in response.text - - # verify backlogged is excluded from the backlog - response = client.get('/') - assert 'my-first-article' in response.text - assert 'my-second-article' not in response.text - assert 'my-third-article' in response.text - - def test_pinned(client): response = create_feed(client, 'feed1.com', [{'title': 'f1-a1', 'date': '2023-10-01 00:00Z'}, {'title': 'f1-a2', 'date': '2023-10-10 00:00Z'}],