RSSQuincieux/gen_feed.py

import requests
import re
import datetime
import xml.dom.minidom
from bs4 import BeautifulSoup
from urllib.parse import urlparse
from zoneinfo import ZoneInfo
from rfeed import *
from dotenv import load_dotenv
import os
import paramiko


class Article:
    def __init__(self, title: str, link: str):
        self.title = title
        self.link = link
        self.text = ""
        self._get_time_and_text()
    
    def _get_time_and_text(self):
        response = requests.get(self.link)
        print(f"   Retrieving {self.link} to get pub time...")
        if response.status_code != 200:
            print("Caramba!")
            return

        soup = BeautifulSoup(response.text, 'html.parser')

        # Handle date
        pt = soup.find("meta", property="article:modified_time")
        pub_time = pt.get("content")
        self.date = datetime.datetime.fromisoformat(pub_time)

        # Handle text
        content = [f"<p>{x.text}</p>" for x in soup.find("div", id="content").find_all("p")]
        self.text = f"<div>{' '.join(content)}</div>"


    def __str__(self):
        return f'{self.title} [{self.date}]'
    
    def __repr__(self):
        return self.__str__()


def upload_to_sftp():

    host = os.environ["FTP_HOST"]
    port = int(os.environ["FTP_PORT"])
    username = os.environ["FTP_USER"]
    password = os.environ["FTP_PASSWORD"]

    transport = paramiko.Transport((host, port))
    transport.connect(username=username, password=password)

    sftp = paramiko.SFTPClient.from_transport(transport)
    sftp.put("mairie_quincieux.xml", f"www/mairie_quincieux.xml")

    sftp.close()
    transport.close()


def generate_feed():
    feed_path = "mairie_quincieux.xml"
    feed_url = os.environ["WEBSITE"] + "/" + feed_path
    main_url = "https://www.quincieux.fr/actus/"

    print(f"Retrieving {main_url}...")
    response = requests.get(main_url)
    if response.status_code != 200:
        print(f"Failed to get url: {response.status_code}")
        return False

    soup = BeautifulSoup(response.text, 'html.parser')
    articles = []

    small_articles = soup.find_all("article")
    print(f"Found {len(small_articles)} articles...")
    for a in small_articles:
        cat = a.find("div", class_="fiche-article__category").find("div").text
        titre = a.find("h2").text
        description = a.find("div", class_="fiche-article__content").find(string=True, recursive=False)
        link = a.find("a").get("href")
        articles.append(Article(f"[{cat}] {titre}", link))
    
    print("Generating feed...")
    items = []
    articles = sorted(articles, key=lambda x: x.date, reverse=True)
    for a in articles:
        item = Item(title = a.title, link= a.link, guid = Guid(a.link+a.date.isoformat()), pubDate = a.date, description = a.text)
        items.append(item)
    feed = Feed(title = "Actus Mairie Quincieux",
        link = feed_url,
        description = "Derniers articles",
        language = "fr-FR",
        lastBuildDate = datetime.datetime.now(datetime.UTC),
        items = items)
    
    print("Writing feed...")
    with open(feed_path, 'w') as out_rss:
        dom = xml.dom.minidom.parseString(feed.rss())
        out_rss.write(dom.toprettyxml())

    print("Uploading feed...")
    upload_to_sftp()

    print("All done!")
    return True


if __name__ == '__main__':
    load_dotenv()
    generate_feed()