First version
This commit is contained in:
commit
ced9c0b54f
3 changed files with 343 additions and 0 deletions
224
.gitignore
vendored
Normal file
224
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,224 @@
|
||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
|
__pycache__/
|
||||||
|
*.py[codz]
|
||||||
|
*$py.class
|
||||||
|
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
share/python-wheels/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.nox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*.cover
|
||||||
|
*.py.cover
|
||||||
|
*.lcov
|
||||||
|
.hypothesis/
|
||||||
|
.pytest_cache/
|
||||||
|
cover/
|
||||||
|
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
local_settings.py
|
||||||
|
db.sqlite3
|
||||||
|
db.sqlite3-journal
|
||||||
|
|
||||||
|
# Flask stuff:
|
||||||
|
instance/
|
||||||
|
.webassets-cache
|
||||||
|
|
||||||
|
# Scrapy stuff:
|
||||||
|
.scrapy
|
||||||
|
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
|
||||||
|
# PyBuilder
|
||||||
|
.pybuilder/
|
||||||
|
target/
|
||||||
|
|
||||||
|
# Jupyter Notebook
|
||||||
|
.ipynb_checkpoints
|
||||||
|
|
||||||
|
# IPython
|
||||||
|
profile_default/
|
||||||
|
ipython_config.py
|
||||||
|
|
||||||
|
# pyenv
|
||||||
|
# For a library or package, you might want to ignore these files since the code is
|
||||||
|
# intended to run in multiple environments; otherwise, check them in:
|
||||||
|
# .python-version
|
||||||
|
|
||||||
|
# pipenv
|
||||||
|
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||||
|
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||||
|
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||||
|
# install all needed dependencies.
|
||||||
|
# Pipfile.lock
|
||||||
|
|
||||||
|
# UV
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
# uv.lock
|
||||||
|
|
||||||
|
# poetry
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||||
|
# poetry.lock
|
||||||
|
# poetry.toml
|
||||||
|
|
||||||
|
# pdm
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||||
|
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
||||||
|
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
||||||
|
# pdm.lock
|
||||||
|
# pdm.toml
|
||||||
|
.pdm-python
|
||||||
|
.pdm-build/
|
||||||
|
|
||||||
|
# pixi
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
||||||
|
# pixi.lock
|
||||||
|
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
||||||
|
# in the .venv directory. It is recommended not to include this directory in version control.
|
||||||
|
.pixi/*
|
||||||
|
!.pixi/config.toml
|
||||||
|
|
||||||
|
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||||
|
__pypackages__/
|
||||||
|
|
||||||
|
# Celery stuff
|
||||||
|
celerybeat-schedule*
|
||||||
|
celerybeat.pid
|
||||||
|
|
||||||
|
# Redis
|
||||||
|
*.rdb
|
||||||
|
*.aof
|
||||||
|
*.pid
|
||||||
|
|
||||||
|
# RabbitMQ
|
||||||
|
mnesia/
|
||||||
|
rabbitmq/
|
||||||
|
rabbitmq-data/
|
||||||
|
|
||||||
|
# ActiveMQ
|
||||||
|
activemq-data/
|
||||||
|
|
||||||
|
# SageMath parsed files
|
||||||
|
*.sage.py
|
||||||
|
|
||||||
|
# Environments
|
||||||
|
.env
|
||||||
|
.envrc
|
||||||
|
.venv
|
||||||
|
env/
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
env.bak/
|
||||||
|
venv.bak/
|
||||||
|
|
||||||
|
# Spyder project settings
|
||||||
|
.spyderproject
|
||||||
|
.spyproject
|
||||||
|
|
||||||
|
# Rope project settings
|
||||||
|
.ropeproject
|
||||||
|
|
||||||
|
# mkdocs documentation
|
||||||
|
/site
|
||||||
|
|
||||||
|
# mypy
|
||||||
|
.mypy_cache/
|
||||||
|
.dmypy.json
|
||||||
|
dmypy.json
|
||||||
|
|
||||||
|
# Pyre type checker
|
||||||
|
.pyre/
|
||||||
|
|
||||||
|
# pytype static type analyzer
|
||||||
|
.pytype/
|
||||||
|
|
||||||
|
# Cython debug symbols
|
||||||
|
cython_debug/
|
||||||
|
|
||||||
|
# PyCharm
|
||||||
|
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||||
|
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||||
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
|
# .idea/
|
||||||
|
|
||||||
|
# Abstra
|
||||||
|
# Abstra is an AI-powered process automation framework.
|
||||||
|
# Ignore directories containing user credentials, local state, and settings.
|
||||||
|
# Learn more at https://abstra.io/docs
|
||||||
|
.abstra/
|
||||||
|
|
||||||
|
# Visual Studio Code
|
||||||
|
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
||||||
|
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
||||||
|
# you could uncomment the following to ignore the entire vscode folder
|
||||||
|
# .vscode/
|
||||||
|
# Temporary file for partial code execution
|
||||||
|
tempCodeRunnerFile.py
|
||||||
|
|
||||||
|
# Ruff stuff:
|
||||||
|
.ruff_cache/
|
||||||
|
|
||||||
|
# PyPI configuration file
|
||||||
|
.pypirc
|
||||||
|
|
||||||
|
# Marimo
|
||||||
|
marimo/_static/
|
||||||
|
marimo/_lsp/
|
||||||
|
__marimo__/
|
||||||
|
|
||||||
|
# Streamlit
|
||||||
|
.streamlit/secrets.toml
|
||||||
|
|
||||||
|
.env
|
||||||
|
|
||||||
|
mairie_quincieux.xml
|
||||||
114
gen_feed.py
Normal file
114
gen_feed.py
Normal file
|
|
@ -0,0 +1,114 @@
|
||||||
|
import requests
|
||||||
|
import re
|
||||||
|
import datetime
|
||||||
|
import xml.dom.minidom
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
from zoneinfo import ZoneInfo
|
||||||
|
from rfeed import *
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
import os
|
||||||
|
import paramiko
|
||||||
|
|
||||||
|
|
||||||
|
class Article:
|
||||||
|
def __init__(self, title: str, link: str):
|
||||||
|
self.title = title
|
||||||
|
self.link = link
|
||||||
|
self.text = ""
|
||||||
|
self._get_time_and_text()
|
||||||
|
|
||||||
|
def _get_time_and_text(self):
|
||||||
|
response = requests.get(self.link)
|
||||||
|
print(f" Retrieving {self.link} to get pub time...")
|
||||||
|
if response.status_code != 200:
|
||||||
|
print("Caramba!")
|
||||||
|
return
|
||||||
|
|
||||||
|
soup = BeautifulSoup(response.text, 'html.parser')
|
||||||
|
|
||||||
|
# Handle date
|
||||||
|
pt = soup.find("meta", property="article:modified_time")
|
||||||
|
pub_time = pt.get("content")
|
||||||
|
self.date = datetime.datetime.fromisoformat(pub_time)
|
||||||
|
|
||||||
|
# Handle text
|
||||||
|
content = [f"<p>{x.text}</p>" for x in soup.find("div", id="content").find_all("p")]
|
||||||
|
self.text = f"<div>{" ".join(content)}</div>"
|
||||||
|
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return f'{self.title} [{self.date}]'
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return self.__str__()
|
||||||
|
|
||||||
|
|
||||||
|
def upload_to_sftp():
|
||||||
|
|
||||||
|
host = os.environ["FTP_HOST"]
|
||||||
|
port = int(os.environ["FTP_PORT"])
|
||||||
|
username = os.environ["FTP_USER"]
|
||||||
|
password = os.environ["FTP_PASSWORD"]
|
||||||
|
|
||||||
|
transport = paramiko.Transport((host, port))
|
||||||
|
transport.connect(username=username, password=password)
|
||||||
|
|
||||||
|
sftp = paramiko.SFTPClient.from_transport(transport)
|
||||||
|
sftp.put("mairie_quincieux.xml", f"www/mairie_quincieux.xml")
|
||||||
|
|
||||||
|
sftp.close()
|
||||||
|
transport.close()
|
||||||
|
|
||||||
|
|
||||||
|
def generate_feed():
|
||||||
|
feed_path = "mairie_quincieux.xml"
|
||||||
|
feed_url = os.environ["WEBSITE"] + "/" + feed_path
|
||||||
|
main_url = "https://www.quincieux.fr/actus/"
|
||||||
|
|
||||||
|
print(f"Retrieving {main_url}...")
|
||||||
|
response = requests.get(main_url)
|
||||||
|
if response.status_code != 200:
|
||||||
|
print(f"Failed to get url: {response.status_code}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
soup = BeautifulSoup(response.text, 'html.parser')
|
||||||
|
articles = []
|
||||||
|
|
||||||
|
small_articles = soup.find_all("article")
|
||||||
|
print(f"Found {len(small_articles)} articles...")
|
||||||
|
for a in small_articles:
|
||||||
|
cat = a.find("div", class_="fiche-article__category").find("div").text
|
||||||
|
titre = a.find("h2").text
|
||||||
|
description = a.find("div", class_="fiche-article__content").find(string=True, recursive=False)
|
||||||
|
link = a.find("a").get("href")
|
||||||
|
articles.append(Article(f"[{cat}] {titre}", link))
|
||||||
|
|
||||||
|
print("Generating feed...")
|
||||||
|
items = []
|
||||||
|
articles = sorted(articles, key=lambda x: x.date, reverse=True)
|
||||||
|
for a in articles:
|
||||||
|
item = Item(title = a.title, link= a.link, guid = Guid(a.link+a.date.isoformat()), pubDate = a.date, description = a.text)
|
||||||
|
items.append(item)
|
||||||
|
feed = Feed(title = "Actus Mairie Quincieux",
|
||||||
|
link = feed_url,
|
||||||
|
description = "Derniers articles",
|
||||||
|
language = "fr-FR",
|
||||||
|
lastBuildDate = datetime.datetime.now(datetime.UTC),
|
||||||
|
items = items)
|
||||||
|
|
||||||
|
print("Writing feed...")
|
||||||
|
with open(feed_path, 'w') as out_rss:
|
||||||
|
dom = xml.dom.minidom.parseString(feed.rss())
|
||||||
|
out_rss.write(dom.toprettyxml())
|
||||||
|
|
||||||
|
print("Uploading feed...")
|
||||||
|
upload_to_sftp()
|
||||||
|
|
||||||
|
print("All done!")
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
load_dotenv()
|
||||||
|
generate_feed()
|
||||||
5
requirements.txt
Normal file
5
requirements.txt
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
rfeed
|
||||||
|
beautifulsoup4
|
||||||
|
requests
|
||||||
|
paramiko
|
||||||
|
python-dotenv
|
||||||
Loading…
Add table
Add a link
Reference in a new issue