Remove backlog and unwrap features (#96)

* remove pop backlog task

* remove backlog from tasks and routes

* remove entry unwrap

* remove backlogged from queries

* remove backlogged from db

* remove test

* remove backlog and unwrap from templates

feedi/models.py

@@ -511,7 +511,6 @@ class Entry(db.Model):
     viewed = sa.Column(sa.TIMESTAMP, index=True)
     favorited = sa.Column(sa.TIMESTAMP, index=True)
     pinned = sa.Column(sa.TIMESTAMP, index=True)
-    backlogged = sa.Column(sa.TIMESTAMP, index=True)
 
     raw_data = sa.orm.deferred(sa.Column(sa.String,
                                          doc="The original entry data received from the feed, as JSON"))
@@ -572,32 +571,8 @@ def fetch_content(self):
             except Exception as e:
                 logger.debug("failed to fetch content %s", e)
 
-    def embedded_links(self):
-        "Return a list of (url, title) for links found in the content_short (excluding hashtags and mentions)."
-        if self.content_short:
-            return [url for (url, text) in scraping.extract_links(self.target_url, self.content_short)
-                    # skip hashtag and profile links
-                    if not text.startswith('#') and not text.startswith('@')]
-        return []
-
-    def is_unwrappable(self):
-        "Return whether there are embedded links that can be extracted from the content_shrot."
-        return bool(self.embedded_links())
-
-    def backlog(self):
-        "Put this entry in the backlog."
-        self.backlogged = datetime.datetime.utcnow()
-        self.pinned = None
-
-    def unbacklog(self):
-        "Pop this entry from the backlog."
-        self.backlogged = None
-        self.viewed = None
-        self.sort_date = datetime.datetime.utcnow()
-        self.fetch_content()
-
     @classmethod
-    def _filtered_query(cls, user_id, hide_seen=False, favorited=None, backlogged=None,
+    def _filtered_query(cls, user_id, hide_seen=False, favorited=None,
                         feed_name=None, username=None, folder=None, older_than=None,
                         text=None):
         """
@@ -619,14 +594,6 @@ def _filtered_query(cls, user_id, hide_seen=False, favorited=None, backlogged=No
         if favorited:
             query = query.filter(cls.favorited.is_not(None))
 
-        if backlogged:
-            query = query.filter(cls.backlogged.is_not(None))
-        elif hide_seen:
-            # exclude backlogged unless explicitly asking for backlog list
-            # or if the hide seen option is disabled.
-            # (abusing the fact that non mixed lists set this flag to false)
-            query = query.filter(cls.backlogged.is_(None))
-
         if feed_name:
             query = query.filter(cls.feed.has(name=feed_name))
 
@@ -664,8 +631,6 @@ def sorted_by(cls, user_id, ordering, start_at, **filters):
 
         if filters.get('favorited'):
             return query.order_by(cls.favorited.desc())
-        if filters.get('backlogged'):
-            return query.order_by(cls.backlogged)
 
         elif ordering == cls.ORDER_RECENCY:
             # reverse chronological order

feedi/routes.py

@@ -16,7 +16,6 @@
 
 @app.route("/users/<username>")
 @app.route("/favorites", defaults={'favorited': True}, endpoint='favorites')
-@app.route("/backlog", defaults={'backlogged': True}, endpoint='backlog')
 @app.route("/folder/<folder>")
 @app.route("/feeds/<feed_name>/entries")
 @app.route("/")
@@ -161,7 +160,6 @@ def autocomplete():
     static_options = [
         ('Home', flask.url_for('entry_list'), 'fas fa-home'),
         ('Favorites', flask.url_for('favorites', favorited=True), 'far fa-star'),
-        ('Backlog', flask.url_for('backlog', favorited=True), 'fa fa-archive'),
         ('Add Feed', flask.url_for('feed_add'), 'fas fa-plus'),
         ('Manage Feeds', flask.url_for('feed_list'), 'fas fa-edit'),
         ('Mastodon login', flask.url_for('mastodon_oauth'), 'fab fa-mastodon'),
@@ -190,7 +188,6 @@ def entry_pin(id):
     else:
         entry.fetch_content()
         entry.pinned = datetime.datetime.utcnow()
-        entry.backlogged = None
     db.session.commit()
 
     # get the new list of pinned based on filters
@@ -220,34 +217,6 @@ def entry_favorite(id):
     return '', 204
 
 
-@app.put("/backlog/<int:id>")
-@login_required
-def entry_backlog_push(id):
-    "Put the entry of the given id in the backlog."
-    entry = db.get_or_404(models.Entry, id)
-    if entry.user_id != current_user.id:
-        flask.abort(404)
-
-    entry.backlog()
-    db.session.commit()
-    return '', 204
-
-
-@app.delete("/backlog/<int:id>")
-@login_required
-def entry_backlog_pop(id):
-    "Remove the entry of the given id from the backlog, sending it back to the home feed."
-    entry = db.get_or_404(models.Entry, id)
-    if entry.user_id != current_user.id:
-        flask.abort(404)
-
-    if entry.backlogged:
-        entry.unbacklog()
-
-    db.session.commit()
-    return '', 204
-
-
 @app.put("/mastodon/favorites/<int:id>")
 @login_required
 def mastodon_favorite(id):
@@ -421,7 +390,6 @@ def feed_delete(feed_name):
     update = db.update(models.Entry)\
         .where((models.Entry.feed_id == feed.id) & (
             models.Entry.favorited.isnot(None) |
-            models.Entry.backlogged.isnot(None) |
             models.Entry.pinned.isnot(None)))\
         .values(feed_id=None)
     db.session.execute(update)
@@ -477,30 +445,6 @@ def entry_add():
         return '', 204
 
 
-@app.post("/entries/<int:id>")
-@login_required
-def entry_unwrap(id):
-    "If the entry has embedded links in its short content, extract the first and render it."
-    entry = db.get_or_404(models.Entry, id)
-    if entry.user_id != current_user.id:
-        flask.abort(404)
-
-    if entry.content_short:
-        # If there's an inline link, "unwrap it", e.g. remove the old entry and put the linked
-        # article entry in its place
-        for link in entry.embedded_links():
-            try:
-                subentry = models.Entry.from_url(current_user.id, link)
-                entry.viewed = datetime.datetime.now()
-                db.session.add(subentry)
-                db.session.commit()
-                return flask.render_template('entry_list_page.html',
-                                             entries=[subentry])
-            except Exception:
-                continue
-    return "Couldn't unwrap", 400
-
-
 @app.get("/entries/<int:id>")
 @login_required
 def entry_view(id):

feedi/scraping.py

@@ -86,13 +86,6 @@ def all_meta(soup):
     return result
 
 
-def extract_links(url, html):
-    soup = BeautifulSoup(html, 'lxml')
-    # checks tag.text so it skips image links
-    links = soup.find_all(lambda tag: tag.name == 'a' and tag.text and 'href' in tag)
-    return [(make_absolute(url, a['href']), a.text) for a in links]
-
-
 def make_absolute(url, path):
     "If `path` is a relative url, join it with the given absolute url."
     if not urllib.parse.urlparse(path).netloc:

feedi/tasks.py

@@ -121,7 +121,6 @@ def delete_old_entries():
         .join(models.Feed.entries)\
         .filter(models.Entry.sort_date < older_than_date,
                 models.Entry.favorited.is_(None),
-                models.Entry.backlogged.is_(None),
                 models.Entry.pinned.is_(None)
                 )\
         .group_by(models.Feed.id)\
@@ -144,7 +143,6 @@ def delete_old_entries():
         q = db.delete(models.Entry)\
               .where(
                   models.Entry.favorited.is_(None),
-                  models.Entry.backlogged.is_(None),
                   models.Entry.pinned.is_(None),
                   models.Entry.feed_id == feed_id,
                   models.Entry.sort_date < min_sort_date,
@@ -169,24 +167,6 @@ def delete_old_entries():
         app.logger.info("Deleted %s old standalone entries from", res.rowcount)
 
 
-@feed_cli.command('backlog')
-@huey_task(crontab(minute='0', hour='0'))
-def pop_backlog():
-    "Periodically pop an entry from the backlog into the home feed."
-    # TODO make this configurable
-    week_ago = datetime.datetime.utcnow() - datetime.timedelta(days=7)
-    backlogged_date = sa.func.min(models.Entry.backlogged).label('backlogged_date')
-    query = db.select(models.Entry)\
-        .group_by(models.Entry.user_id)\
-        .having(backlogged_date < week_ago)
-
-    for entry in db.session.scalars(query):
-        entry.unbacklog()
-        app.logger.info("Popped from user %s backlog: %s ", entry.user_id, entry.target_url)
-
-    db.session.commit()
-
-
 @feed_cli.command('debug')
 @click.argument('url')
 def debug_feed(url):

feedi/templates/base.html

@@ -24,8 +24,6 @@
                 {% endfor %}
                 <a class="dropdown-item {% if filters.favorited %}is-active{% endif %}" href="{{ url_for('favorites') }}">
                     <icon class="icon"><i class="far fa-star"></i></icon> favorites</a>
-                <a class="dropdown-item {% if filters.backlogged %}is-active{% endif %}" href="{{ url_for('backlog') }}">
-                    <icon class="icon"><i class="fas fa-archive"></i></icon> backlog</a>
                 <hr class="dropdown-divider"/>
                 {% endif %}
                 {{ self.sidebar_right() }}
@@ -86,14 +84,6 @@ <h1 class="title is-4">feedi</h1>
                                 </div>
                             </div>
                         </li>
-                        <li>
-                            <div class="level-left {% if filters.backlogged %}is-active{% endif %}">
-                                <div class="level-item">
-                                    <a href="{{ url_for('backlog') }}"><icon class="icon"><i class="fas fa-archive"></i></icon>
-                                        backlog</a>
-                                </div>
-                            </div>
-                        </li>
                         </ul>
                 </aside>
                 {% endblock %}

feedi/templates/entry_commands.html

@@ -12,11 +12,6 @@
 {% if entry.content_url and entry.id and request.path != url_for('entry_view', id=entry.id) %}
 <a hx-boost="true" href="{{ url_for('entry_view', id=entry.id) }}" class="dropdown-item"><span class="icon"><i class="fas fa-book-reader"></i></span> View in reader</a>
 {% endif %}
-{% if not entry.backlogged %}
-<a hx-put="{{ url_for('entry_backlog_push', id=entry.id) }}" class="dropdown-item"
-   _="on htmx:afterRequest remove the closest .feed-entry"
-><span class="icon"><i class="fas fa-archive"></i></span> Send to backlog</a>
-{% endif %}
 <hr class="dropdown-divider">
 <a class="dropdown-item" _="on click writeText('{{ entry.content_url }}') into the navigator's clipboard"><span class="icon"><i class="fas fa-link"></i></span> Copy URL</a>
 

feedi/templates/entry_header.html

@@ -83,19 +83,6 @@
            href="{{ entry.comments_url}}" target="_blank"
         ><i class="fas fa-comment-alt"></i></a>
         {% endif %}
-        {% if entry.backlogged %}
-        <a tabindex="-1" class="level-item icon hover-icon is-white is-rounded" title="Pop from backlog"
-           _="on click remove the closest .feed-entry"
-           hx-delete="{{ url_for('entry_backlog_pop', id=entry.id )}}"
-        ><i class="fas fa-undo"></i></a>
-        {% elif entry.is_unwrappable() %}
-        <a tabindex="-1" class="level-item icon hover-icon is-white is-rounded" title="Unwrap"
-           hx-post="{{ url_for('entry_unwrap', id=entry.id )}}"
-           hx-target="closest .feed-entry"
-           hx-swap="outerHTML"
-           _ = "on click set x to the closest .feed-entry then set x.style.opacity to 0.5"
-        ><i class="fas fa-envelope-open-text"></i></a>
-        {% endif %}
         <div class="dropdown is-right"
              _="on intersection(intersecting) having margin '0px 0px -50% 0px'
                     if intersecting remove .is-up else add .is-up -- show dropup up or dropdown depending position relative to middle of screen">

feedi/templates/entry_list_page.html

@@ -83,19 +83,6 @@
             <i class="fas fa-comment-alt"></i>
         </a>
         {% endif %}
-        {% if entry.backlogged %}
-        <a tabindex="-1" class="level-item icon  is-white is-rounded" title="Pop from backlog"
-           _="on click remove the closest .feed-entry"
-           hx-delete="{{ url_for('entry_backlog_pop', id=entry.id )}}"
-        ><i class="fas fa-undo"></i></a>
-        {% elif entry.is_unwrappable() %}
-        <a tabindex="-1" class="level-item icon is-white is-rounded" title="Unwrap"
-           hx-post="{{ url_for('entry_unwrap', id=entry.id )}}"
-           hx-target="closest .feed-entry"
-           hx-swap="outerHTML"
-           _ = "on click set x to the closest .feed-entry then set x.style.opacity to 0.5"
-        ><i class="fas fa-envelope-open-text"></i></a>
-        {% endif %}
 
         <a class="icon is-white is-rounded level-item"
            tabindex="-1"

migrations/versions/f11ef7901ceb_remove_backlogged.py

@@ -0,0 +1,36 @@
+"""remove backlogged
+
+Revision ID: f11ef7901ceb
+Revises: fc7cea04e4ca
+Create Date: 2024-05-24 16:26:22.169909
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'f11ef7901ceb'
+down_revision: Union[str, None] = 'fc7cea04e4ca'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('entries', schema=None) as batch_op:
+        batch_op.drop_index('ix_entries_backlogged')
+        batch_op.drop_column('backlogged')
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('entries', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('backlogged', sa.TIMESTAMP(), nullable=True))
+        batch_op.create_index('ix_entries_backlogged', ['backlogged'], unique=False)
+
+    # ### end Alembic commands ###

tests/test_routes.py

@@ -202,50 +202,6 @@ def test_favorites(client):
     assert response.text.find('my-second-article') < response.text.find('my-third-article')
 
 
-def test_backlog(client):
-    # add feed with a couple of articles
-    feed_domain = 'feed1.com'
-    response = create_feed(client, feed_domain, [{'title': 'my-third-article', 'date': '2023-11-10 00:00Z'},
-                                                 {'title': 'my-second-article',
-                                                     'date': '2023-10-10 00:00Z'},
-                                                 {'title': 'my-first-article', 'date': '2023-10-01 00:00Z'}])
-    entry_ids = extract_entry_ids(response)
-
-    # backlog the 3rd, then the 2nd
-    response = client.put(f'/backlog/{entry_ids[0]}')
-    assert response.status_code == 204
-    response = client.put(f'/backlog/{entry_ids[1]}')
-    assert response.status_code == 204
-
-    # verify backlogged is excluded from home list
-    response = client.get('/backlog')
-    assert 'my-first-article' not in response.text
-    assert 'my-second-article' in response.text
-    assert 'my-third-article' in response.text
-
-    # verify backlogged is included in the backlog, in the order they were added
-    response = client.get('/')
-    assert 'my-first-article' in response.text
-    assert 'my-second-article' not in response.text
-    assert 'my-third-article' not in response.text
-
-    # pop from backlog
-    response = client.delete(f'/backlog/{entry_ids[0]}')
-    assert response.status_code == 204
-
-    # verify backlogged is included in home list
-    response = client.get('/backlog')
-    assert 'my-first-article' not in response.text
-    assert 'my-second-article' in response.text
-    assert 'my-third-article' not in response.text
-
-    # verify backlogged is excluded from the backlog
-    response = client.get('/')
-    assert 'my-first-article' in response.text
-    assert 'my-second-article' not in response.text
-    assert 'my-third-article' in response.text
-
-
 def test_pinned(client):
     response = create_feed(client, 'feed1.com', [{'title': 'f1-a1', 'date': '2023-10-01 00:00Z'},
                                                  {'title': 'f1-a2', 'date': '2023-10-10 00:00Z'}],