From 62652b61aead1426ff5e0e7dcbb33236bbe94638 Mon Sep 17 00:00:00 2001 From: Robert van Barlingen Date: Tue, 16 Jul 2024 09:16:14 +0200 Subject: [PATCH 01/10] adds Dockerfile, docker-compose.yml, requirements.txt and changes the code a bit to make it more compatible with docker --- .dockerignore | 19 +++++++++++++++++++ .gitignore | 1 + Dockerfile | 6 ++++++ docker-compose.yml | 7 +++++++ requirements.txt | 3 +++ rss2newsletter | 3 +-- rss2newsletter.conf | 4 ++-- 7 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 requirements.txt diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8c4b167 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/.old +**/bin +**/docker-compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +processed_entries.txt +rss2newsletter.egg-info +LICENSE +README.md \ No newline at end of file diff --git a/.gitignore b/.gitignore index 91cf2ee..2f00311 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ processed_entries.txt *.swp *.swo +*.old rss2newsletter.egg-info dist diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c7ca463 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3-alpine +WORKDIR /rss2newsletter +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . +CMD ["python", "-u", "./rss2newsletter"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..2f98044 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,7 @@ +services: + rss2newsletter: + image: rss2newsletter + container_name: rss2newsletter + restart: unless-stopped + volumes: + - ./rss2newsletter.conf:/rss2newsletter/rss2newsletter.conf \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3c9f73e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +requests +feedparser +lxml \ No newline at end of file diff --git a/rss2newsletter b/rss2newsletter index ef51404..365b2ff 100755 --- a/rss2newsletter +++ b/rss2newsletter @@ -111,8 +111,7 @@ class rss2newsletter: feed = feedparser.parse(self.config["FEED"]["URL"]) # In case of failure if hasattr(feed, "bozo_exception"): - print(f"Error fetching RSS from: {self.config['FEED']['URL']}") - return None + raise Exception(f"Error fetching RSS from: {self.config['FEED']['URL']}") return feed diff --git a/rss2newsletter.conf b/rss2newsletter.conf index dd649a7..093155f 100644 --- a/rss2newsletter.conf +++ b/rss2newsletter.conf @@ -1,6 +1,6 @@ [FEED] # Full URL to your website's feed -URL = https://elliotonsecurity.com/atom.xml +URL = YOUR_FEED_URL # How often to check for new feed entries in seconds POLL_INTERVAL = 300 @@ -14,7 +14,7 @@ PROCESSED_ENTRIES_FILE = processed_entries.txt URL = http://localhost:9000 # Credentials -USERNAME = ElliotKillick +USERNAME = YOUR_USERNAME PASSWORD = YOUR_PASSWORD # The ID of your "rss2newsletter" list (create this list in listmonk) From d6b5bcc1de19db744420da8ca9b6df6853fc37c5 Mon Sep 17 00:00:00 2001 From: Robert van Barlingen Date: Tue, 16 Jul 2024 09:31:55 +0200 Subject: [PATCH 02/10] adds volume for newsletter_template.html to docker-compose --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2f98044..6f184e6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,4 +4,5 @@ services: container_name: rss2newsletter restart: unless-stopped volumes: - - ./rss2newsletter.conf:/rss2newsletter/rss2newsletter.conf \ No newline at end of file + - ./rss2newsletter.conf:/rss2newsletter/rss2newsletter.conf + - ./newsletter_template.html:/rss2newsletter/newsletter_template.html \ No newline at end of file From fc23213e4b132da208154e2ab38c577273b2f7aa Mon Sep 17 00:00:00 2001 From: Robert van Barlingen <10343246+Robert-van-Barlingen@users.noreply.github.com> Date: Tue, 30 Jul 2024 08:04:59 +0200 Subject: [PATCH 03/10] Create main.yml Creates main.yml github action for automatically building docker container and pushing to dockerhub. --- .github/workflows/main.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..69f94d6 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,26 @@ +name: ci + +on: + push: + branches: + - "main" + +jobs: + build: + runs-on: ubuntu-latest + steps: + - + name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - + name: Build and push + uses: docker/build-push-action@v6 + with: + push: true + tags: ${{ vars.DOCKERHUB_USERNAME }}/rss2newsletter:latest From fa0df7ddea2b0623b8f69674220c4b54ade5a1e0 Mon Sep 17 00:00:00 2001 From: Robert van Barlingen Date: Wed, 31 Jul 2024 17:43:53 +0200 Subject: [PATCH 04/10] changes hardcoded listmonk list ID to list ID from config --- rss2newsletter | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rss2newsletter b/rss2newsletter index 365b2ff..2bf7d46 100755 --- a/rss2newsletter +++ b/rss2newsletter @@ -244,7 +244,7 @@ class rss2newsletter: json_data = { "name": name, "subject": name, - "lists": [1], + "lists": [int(self.config["LISTMONK"]["LIST_ID"])], "content_type": "richtext", "body": body, "messenger": "email", From 02c485a92661d5546a91692c1ff2349284b54c90 Mon Sep 17 00:00:00 2001 From: Robert van Barlingen Date: Wed, 31 Jul 2024 18:18:42 +0200 Subject: [PATCH 05/10] adds volume for rss2newsletter-data in order to make the processed_entries.txt file persistent --- docker-compose.yml | 8 +++++++- rss2newsletter.conf | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6f184e6..1b01bc2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,4 +5,10 @@ services: restart: unless-stopped volumes: - ./rss2newsletter.conf:/rss2newsletter/rss2newsletter.conf - - ./newsletter_template.html:/rss2newsletter/newsletter_template.html \ No newline at end of file + - ./newsletter_template.html:/rss2newsletter/newsletter_template.html + - type: volume + source: rss2newsletter-data + target: /rss2newsletter/data + +volumes: + rss2newsletter-data: \ No newline at end of file diff --git a/rss2newsletter.conf b/rss2newsletter.conf index 093155f..c194418 100644 --- a/rss2newsletter.conf +++ b/rss2newsletter.conf @@ -6,7 +6,7 @@ URL = YOUR_FEED_URL POLL_INTERVAL = 300 # rss2newsletter uses this file to keep track of new feed entries -PROCESSED_ENTRIES_FILE = processed_entries.txt +PROCESSED_ENTRIES_FILE = data/processed_entries.txt [LISTMONK] From 6eab0d85275e3ef3f6421aaaa9624203cea1e58c Mon Sep 17 00:00:00 2001 From: Robert van Barlingen Date: Tue, 6 Aug 2024 10:58:54 +0200 Subject: [PATCH 06/10] adds support for additional rss tags to be imported into the newsletter template. --- rss2newsletter | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/rss2newsletter b/rss2newsletter index 2bf7d46..a7fc644 100755 --- a/rss2newsletter +++ b/rss2newsletter @@ -100,7 +100,7 @@ class rss2newsletter: for new_entry in self.check_for_new_entries( processed_entries_last_update, feed.entries ): - campaign_id = self.create_newsletter(new_entry.link, new_entry.title) + campaign_id = self.create_newsletter(new_entry) send_successful = self.send_newsletter(campaign_id) if send_successful: self.update_processed_entries_file(new_entry.link) @@ -165,13 +165,13 @@ class rss2newsletter: if entry.link not in proceseed_entries_last_update: yield entry - def create_newsletter(self, link: str, title: str) -> int | None: + def create_newsletter(self, feed: feedparser.FeedParserDict) -> int | None: """Create newsletter with content and add campaign to Listmonk""" - print("Creating newsletter for:", title) - return self.create_campaign(title, self.create_content(link, title)) + print("Creating newsletter for:", feed.title) + return self.create_campaign(feed.title, self.create_content(feed)) - def create_content(self, link: str, title: str) -> str: + def create_content(self, feed: feedparser.FeedParserDict) -> str: """Create content to be used as body of newsletter""" with open( @@ -179,10 +179,18 @@ class rss2newsletter: ) as f: content = f.read() - content = content.replace("LINK_HERE", link) - content = content.replace("TITLE_HERE", title) - - og_image = self.get_og_image(self.fetch_url(link)) + content = content.replace("LINK_HERE", feed.link) + content = content.replace("TITLE_HERE", feed.title) + content = content.replace("SUMMARY_HERE", feed.summary) + content = content.replace("PUBLISHED_HERE", time.strftime("%d-%m-%Y", feed.published_parsed)) + content = content.replace("CONTENT_HERE", feed.content[0].value) + content = content.replace("AUTHOR_NAME_HERE", feed.author_detail.name) + content = content.replace("AUTHOR_EMAIL_HERE", feed.author_detail.email) + tags_string = ", ".join([tag.term for tag in feed.tags]) + content = content.replace("TAGS_HERE", tags_string) + content = content.replace("MEDIA_HERE", feed.media_content[0]["url"]) + + og_image = self.get_og_image(self.fetch_url(feed.link)) if og_image: content = content.replace("IMAGE_HERE", og_image) else: From bfdfe8b6caba489a49154b24e3364273a2774f97 Mon Sep 17 00:00:00 2001 From: Robert van Barlingen Date: Tue, 6 Aug 2024 11:14:43 +0200 Subject: [PATCH 07/10] adds section to README detailing the supported RSS tags in the newsletter template --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index c38a483..d05ccb2 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,20 @@ options: Program configuration file (default: rss2newsletter.conf) ``` +## Supported RSS tags in the newsletter template +You can design your own newsletter template by modifying the [newsletter template file](https://raw.githubusercontent.com/ElliotKillick/rss2newsletter/main/newsletter_template.html). Certain keywords in the template are automatically replaced by their corresponding RSS tags. + +| **template keyword** | **RSS tag** | +| :---: | :---: | +| LINK_HERE | link | +| TITLE_HERE | title | +| SUMMARY_HERE | description | +| PUBLISHED_HERE | pubdate | +| CONTENT_HERE | content | +| AUTHOR_NAME_HERE | author (name) | +| AUTHOR_EMAIL_HERE | author (email) | +| TAGS_HERE | category (as comma separated string) | +| MEDIA_HERE | media (url) | ## Support the Author From 4afc02de2cae632e34e51d32fc08649c9f907b90 Mon Sep 17 00:00:00 2001 From: Robert van Barlingen Date: Fri, 16 Aug 2024 17:38:05 +0200 Subject: [PATCH 08/10] for every feed attribute, first check if the attribute is present before attempting to replace in the newsletter template --- rss2newsletter | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/rss2newsletter b/rss2newsletter index a7fc644..d286ab9 100755 --- a/rss2newsletter +++ b/rss2newsletter @@ -179,16 +179,26 @@ class rss2newsletter: ) as f: content = f.read() - content = content.replace("LINK_HERE", feed.link) - content = content.replace("TITLE_HERE", feed.title) - content = content.replace("SUMMARY_HERE", feed.summary) - content = content.replace("PUBLISHED_HERE", time.strftime("%d-%m-%Y", feed.published_parsed)) - content = content.replace("CONTENT_HERE", feed.content[0].value) - content = content.replace("AUTHOR_NAME_HERE", feed.author_detail.name) - content = content.replace("AUTHOR_EMAIL_HERE", feed.author_detail.email) - tags_string = ", ".join([tag.term for tag in feed.tags]) - content = content.replace("TAGS_HERE", tags_string) - content = content.replace("MEDIA_HERE", feed.media_content[0]["url"]) + + if hasattr(feed, "link"): + content = content.replace("LINK_HERE", feed.link) + if hasattr(feed, "title"): + content = content.replace("TITLE_HERE", feed.title) + if hasattr(feed, "summary"): + content = content.replace("SUMMARY_HERE", feed.summary) + if hasattr(feed, "published_parsed"): + content = content.replace("PUBLISHED_HERE", time.strftime("%d-%m-%Y", feed.published_parsed)) + if hasattr(feed, "content") and len(feed.content) > 0: + content = content.replace("CONTENT_HERE", feed.content[0].value) + if hasattr(feed, "author_detail") and hasattr(feed.author_detail, "name"): + content = content.replace("AUTHOR_NAME_HERE", feed.author_detail.name) + if hasattr(feed, "author_detail") and hasattr(feed.author_detail, "email"): + content = content.replace("AUTHOR_EMAIL_HERE", feed.author_detail.email) + if hasattr(feed, "tags"): + tags_string = ", ".join([tag.term for tag in feed.tags]) + content = content.replace("TAGS_HERE", tags_string) + if hasattr(feed, "media_content") and len(feed.media_content) > 0: + content = content.replace("MEDIA_HERE", feed.media_content[0]["url"]) og_image = self.get_og_image(self.fetch_url(feed.link)) if og_image: From c19232794478d27d96e2d2d06a9204eb93a4ba12 Mon Sep 17 00:00:00 2001 From: Robert van Barlingen Date: Sat, 17 Aug 2024 00:34:49 +0200 Subject: [PATCH 09/10] replaces feed.author_detail with feed.author for inserting author name and email into the template --- rss2newsletter | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/rss2newsletter b/rss2newsletter index d286ab9..06e71a2 100755 --- a/rss2newsletter +++ b/rss2newsletter @@ -190,16 +190,22 @@ class rss2newsletter: content = content.replace("PUBLISHED_HERE", time.strftime("%d-%m-%Y", feed.published_parsed)) if hasattr(feed, "content") and len(feed.content) > 0: content = content.replace("CONTENT_HERE", feed.content[0].value) - if hasattr(feed, "author_detail") and hasattr(feed.author_detail, "name"): - content = content.replace("AUTHOR_NAME_HERE", feed.author_detail.name) - if hasattr(feed, "author_detail") and hasattr(feed.author_detail, "email"): - content = content.replace("AUTHOR_EMAIL_HERE", feed.author_detail.email) + if hasattr(feed, "author"): + email = feed.author.split(' ')[0] + content = content.replace("AUTHOR_EMAIL_HERE", email) + start_index_author = feed.author.find('(') + 1 + end_index_author = feed.author.find(')') + if start_index_author != -1 and end_index_author != -1: + author = feed.author[start_index_author:end_index_author] + content = content.replace("AUTHOR_NAME_HERE", author) if hasattr(feed, "tags"): tags_string = ", ".join([tag.term for tag in feed.tags]) content = content.replace("TAGS_HERE", tags_string) if hasattr(feed, "media_content") and len(feed.media_content) > 0: content = content.replace("MEDIA_HERE", feed.media_content[0]["url"]) + print(feed) + og_image = self.get_og_image(self.fetch_url(feed.link)) if og_image: content = content.replace("IMAGE_HERE", og_image) From 8ae183dc737caf7e0ff931bf93cccb95d5c3d6aa Mon Sep 17 00:00:00 2001 From: Robert van Barlingen Date: Sat, 17 Aug 2024 10:14:16 +0200 Subject: [PATCH 10/10] removes debug print statement --- rss2newsletter | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/rss2newsletter b/rss2newsletter index 06e71a2..c855c7c 100755 --- a/rss2newsletter +++ b/rss2newsletter @@ -193,10 +193,10 @@ class rss2newsletter: if hasattr(feed, "author"): email = feed.author.split(' ')[0] content = content.replace("AUTHOR_EMAIL_HERE", email) - start_index_author = feed.author.find('(') + 1 + start_index_author = feed.author.find('(') end_index_author = feed.author.find(')') if start_index_author != -1 and end_index_author != -1: - author = feed.author[start_index_author:end_index_author] + author = feed.author[start_index_author + 1:end_index_author] content = content.replace("AUTHOR_NAME_HERE", author) if hasattr(feed, "tags"): tags_string = ", ".join([tag.term for tag in feed.tags]) @@ -204,8 +204,6 @@ class rss2newsletter: if hasattr(feed, "media_content") and len(feed.media_content) > 0: content = content.replace("MEDIA_HERE", feed.media_content[0]["url"]) - print(feed) - og_image = self.get_og_image(self.fetch_url(feed.link)) if og_image: content = content.replace("IMAGE_HERE", og_image)