linuxgaming

This is my own Linux gaming aggregate web app I built for personal use.
git clone git://git.beardyjay.co.uk/linuxgaming
Log | Files | Refs | README

commit 1a952985649dee92fcbfa5603faea6bc315e54f6
parent 6e63bda99c27d108a62a3dbeb3b5c35bfd76246b
Author: Jay Scott <jay@beardyjay.co.uk>
Date:   Wed, 18 Jul 2018 20:18:33 +0100

more refactoring

Diffstat:
Mconfig/feed_config.yaml | 11+++++------
Mlinuxgaming/__init__.py | 6+++---
Mlinuxgaming/database.py | 6+++---
Mlinuxgaming/update.py | 103++++++++++++++++++++++++++++++++++++-------------------------------------------
Mlinuxgaming/util.py | 36+++++++++++++++++++++++++++++++++---
5 files changed, 91 insertions(+), 71 deletions(-)

diff --git a/config/feed_config.yaml b/config/feed_config.yaml @@ -18,14 +18,14 @@ NuSuey: twitch: website: "https://www.twitch.tv/nusuey/" twitch_id: "7863182" - + HexDSL: icon: "hexdsl.png" about: "Linux Gaming Rambles, shows and streams presented by HexDSL. Apparently you don't get a lot of characters in this box. so I'll just say that I am HexDSL (Sexy Hexy) and I make videos about gaming using Linux. I also Host the .XPenguin PodCast/VideoCast that people seem to like :) You can find more of me on twitch.tv/hexdsl/ thanks for taking the time to read this." youtube: website: "https://www.youtube.com/user/hexdsl/" channel_id: "UCRE3NFNtdjR96-H4QG4U1Fg" - twitch: + twitch: website: "https://www.twitch.tv/hexdsl/" twitch_id: "40594186" @@ -35,7 +35,7 @@ TheLinuxGamer: youtube: website: "https://www.youtube.com/user/tuxreviews/" channel_id: "UCv1Kcz-CuGM6mxzL3B1_Eiw" - + GamingOnLinux: icon: "gol.png" about: "A little bit of information on who GamingOnLinux are, we are just a small group of passionate Linux fans who wish to bring you nothing but the best in Linux gaming news. Started by liamdawe on his own in 2009 on a .info address, we later switched to using a .com domain name." @@ -46,7 +46,7 @@ GamingOnLinux: twitch: website: "https://www.twitch.tv/gamingonlinux/" twitch_id: "50905707" - + BLGP: icon: "blgp.png" about: "The Best Linux Games Podcast features only the best news, reviews, and deals exclusively focusing on titles available for the Linux OS. Heavily focuses on Steam titles. Pushed onto teh interwebs every Saturday and hosted by Skookiesprite, the podcast brings an entire week's worth of news, reviews, tech tips, and the best-loved DEALS! Segment straight into your brain." @@ -80,4 +80,4 @@ XPenguin: rss: website: "http://xpenguin.club/" type: "podcast" - url: "http://xpenguin.club/rss.xml"- \ No newline at end of file + url: "http://xpenguin.club/rss.xml" diff --git a/linuxgaming/__init__.py b/linuxgaming/__init__.py @@ -46,13 +46,13 @@ def create_app(): def internal_error(error): app.logger.error('internal error %s', error) return render_template( - "message.html", icon="frown", msg="Something went wrong!"), 500 + "message.html", msg="Something went wrong!"), 500 @app.errorhandler(404) - def page_not_found(): + def page_not_found(page): app.logger.info('page not found') return render_template( - "message.html", icon="frown", msg="I think you are lost!"), 404 + "message.html", msg="I think you are lost!"), 404 @app.template_filter('strftime') def _jinja2_filter_datetime(date): diff --git a/linuxgaming/database.py b/linuxgaming/database.py @@ -1,4 +1,4 @@ -from flask import current_app +from flask import current_app, abort def find_all(query={}): @@ -7,7 +7,7 @@ def find_all(query={}): d = current_app.mongo.db.items.find(query).sort('date', -1) except Exception as e: current_app.logger.error('DB replace error %s', e) - return False + abort(500) return d @@ -18,6 +18,6 @@ def replace_one(query, data, upsert=True): current_app.mongo.db.items.replace_one(query, data, upsert) except Exception as e: current_app.logger.error('DB replace error %s', e) - return False + abort(500) return True diff --git a/linuxgaming/update.py b/linuxgaming/update.py @@ -1,52 +1,60 @@ from googleapiclient.discovery import build from twitch import TwitchClient from flask import Blueprint, render_template, current_app -from . import database -import json -import requests -import feedparser import dateutil.parser +from . import database from . import util bp = Blueprint('update', __name__, url_prefix='/update') -def parse(url): - return feedparser.parse(url).entries - - @bp.route('/rss', methods=["GET"]) def rss_update(): + # load sources config feed_config = util.load_yaml() for section in feed_config: + # if it does not have an rss section skip if 'rss' not in feed_config[section]: continue current_app.logger.info('[RSS] Updating %s', section) - feeds = parse(feed_config[section]['rss']['url']) + + # parse the source url rss feed + feeds = util.feed_parse(feed_config[section]['rss']['url']) + + # check for errors + if feeds is None: + continue for feed in feeds: - trimtitle = feed.title[0:150] + # if not title, then just skip + if hasattr(feed, 'title'): + trimmed_title = feed.title[0:150] + else: + continue + # some feeds dont have a description, RSS 2.0 if not hasattr(feed, 'description'): description = "" else: description = feed.description + # construct db item data = { "name": section, "icon": feed_config[section]['icon'], - "title": trimtitle, + "title": trimmed_title, "description": description, "url": feed.link, "type": feed_config[section]['rss']['type'], "date": dateutil.parser.parse(feed.updated) } - database.replace_one({'title': trimtitle}, data) + # insert based on title + database.replace_one({'title': trimmed_title}, data) return render_template("message.html", msg="RSS feeds updated!") @@ -61,6 +69,7 @@ def twitch_update(): continue current_app.logger.info('[TWITCH] Updating %s', section) + twitch_channelid = feed_config[section]['twitch']['twitch_id'] client = TwitchClient( @@ -77,11 +86,11 @@ def twitch_update(): ) for search_results in videos: - trimtitle = search_results['title'][0:150] + trimmed_title = search_results['title'][0:150] data = { "name": section, "icon": feed_config[section]['icon'], - "title": trimtitle, + "title": trimmed_title, "description": search_results['description'], "url": search_results['url'], "type": "twitch", @@ -96,10 +105,6 @@ def twitch_update(): @bp.route('/youtube', methods=["GET"]) def youtube_update(): - key = current_app.config['YOUTUBE_APIKEY'] - youtube_api = 'youtube' - api_version = 'v3' - feed_config = util.load_yaml() for section in feed_config: @@ -108,9 +113,11 @@ def youtube_update(): youtube_channel_id = feed_config[section]['youtube']['channel_id'] - youtube = build(youtube_api, api_version, developerKey=key) + youtube = build( + 'youtube', 'v3', developerKey=current_app.config['YOUTUBE_APIKEY']) current_app.logger.info('[YOUTUBE] Updating %s', section) + search_response = youtube.search().list( q="", channelId=youtube_channel_id, @@ -119,28 +126,26 @@ def youtube_update(): maxResults=5).execute() for search_result in search_response.get('items', []): - trimtitle = search_result['snippet']['title'][0:150] - if search_result['id']['kind'] == 'youtube#video': - data = { - "name": - section, - "icon": - feed_config[section]['icon'], - "title": - trimtitle, - "description": - search_result['snippet']['description'], - "type": - "youtube", - "url": - "https://www.youtube.com/watch?v=" + - search_result['id']['videoId'], - "date": - dateutil.parser.parse( - search_result['snippet']['publishedAt']) - } - - database.replace_one({'title': trimtitle}, data) + trimmed_title = search_result['snippet']['title'][0:150] + data = { + "name": + section, + "icon": + feed_config[section]['icon'], + "title": + trimmed_title, + "description": + search_result['snippet']['description'], + "type": + "youtube", + "url": + "https://www.youtube.com/watch?v=" + + search_result['id']['videoId'], + "date": + dateutil.parser.parse(search_result['snippet']['publishedAt']) + } + + database.replace_one({'title': trimmed_title}, data) return render_template("message.html", msg="Youtube API updated!") @@ -153,9 +158,7 @@ def gog_update(): count = 1 while count < 51: query = "mediaType=game&system=Linux&limit=50&page=" + str(count) - game_data = get_gog_info(query) - if game_data is None: - return render_template("message.html", msg="GoG query error") + game_data = util.get_gog_info(query) for search_result in game_data['products']: @@ -196,15 +199,3 @@ def gog_update(): return render_template( "message.html", icon="smile", msg="GoG games updated!") - - -def get_gog_info(query): - - gog_api_url = "https://embed.gog.com/games/ajax/filtered?" - - response = requests.get(gog_api_url + query) - - if response.status_code == 200: - return json.loads(response.content.decode('utf-8')) - else: - return None diff --git a/linuxgaming/util.py b/linuxgaming/util.py @@ -1,13 +1,43 @@ import yaml -from flask import (current_app) +import feedparser +import json +import requests +from flask import (current_app, abort) def load_yaml(): """Return the YAML parsed config file.""" + try: with open('config/feed_config.yaml', 'r') as ymlfile: cfg = yaml.load(ymlfile) - except yaml.YAMLError as exc: - current_app.logger.error('YAML read error %s', exc) + except (yaml.YAMLError, FileNotFoundError) as e: + current_app.logger.error('YAML read error %s', e) + abort(500) return cfg + + +def feed_parse(url): + + # parse the feed and get the results + res = feedparser.parse(url) + + if res.entries: + return res.entries + else: + current_app.logger.error('FEED parse error %s', url) + + return None + + +def get_gog_info(query): + + gog_api_url = "https://embed.gog.com/games/ajax/filtered?" + + response = requests.get(gog_api_url + query) + + if response.status_code == 200: + return json.loads(response.content.decode('utf-8')) + else: + abort(500)