Mercurial > hg > Feedworm
changeset 166:04c3b9796b89
feedparser uses the proxy now if one is configured. To implement this the FeedUpdater had to change a bit - sqlalchemy backend is not yet refactored.
| author | dirk |
|---|---|
| date | Sat, 03 Sep 2011 04:12:35 +0200 |
| parents | a5bda54309ba |
| children | a3c945ce434c |
| files | backend/AbstractFeedUpdater.py backend/couchdb/CouchDbBackend.py backend/couchdb/Feed.py backend/couchdb/FeedUpdater.py |
| diffstat | 4 files changed, 39 insertions(+), 27 deletions(-) [+] |
line wrap: on
line diff
--- a/backend/AbstractFeedUpdater.py Mon Aug 29 04:12:01 2011 +0200 +++ b/backend/AbstractFeedUpdater.py Sat Sep 03 04:12:35 2011 +0200 @@ -2,6 +2,7 @@ from datetime import datetime import feedparser import logging +from urllib2 import ProxyHandler STATUS_ERROR = 400 log = logging.getLogger("FeedUpdater") @@ -12,19 +13,28 @@ Subclasses need to implement creating and storing the new feed entries. ''' - def __init__(self, feed): - self.feed = feed + @staticmethod + def parseFeed(url): + proxy = ProxyHandler( {"http":"http://your.proxy.here:8080/"} ) + return feedparser.parse(url, handlers = [proxy]) - def update(self, feedDict=None): - log.info("updating " + self.feed.rss_url) - if feedDict is None: - result = self._retrieveFeed() - else: - result = feedDict + def __init__(self, preferences): + self.preferences = preferences + + def update(self, feed): + self.feed = feed + log.info("updating " + feed.rss_url) + result = self._retrieveFeed() self._processEntries(result) + self._setFeedTitle(result) def _retrieveFeed(self): - result = feedparser.parse(self.feed.rss_url) + if self.preferences.isProxyConfigured(): + proxyUrl = "http://%s:%i" % (self.preferences.proxyHost(), self.preferences.proxyPort()) + proxyHandler = ProxyHandler({"http" : proxyUrl}) + result = feedparser.parse(self.feed.rss_url, handlers=[proxyHandler]) + else: + result = feedparser.parse(self.feed.rss_url) # bozo flags if a feed is well-formed. # if result["bozo"] > 0: # raise FeedUpdateException() @@ -58,6 +68,13 @@ def _incrementFeedUpdateDate(self): raise Exception("_incrementNextUpdateDate is abstract, subclasses must override") + def _setFeedTitle(self, feedDict): + if self.feed.title is None: + if feedDict.feed.has_key("title"): + self.feed.title = feedDict.feed.title + else: + self.feed.title = self.feed.rss_url + class FeedUpdateException(Exception): pass
--- a/backend/couchdb/CouchDbBackend.py Mon Aug 29 04:12:01 2011 +0200 +++ b/backend/couchdb/CouchDbBackend.py Sat Sep 03 04:12:35 2011 +0200 @@ -67,7 +67,9 @@ raise Exception("not yet implemented") def createFeed(self, url): - FeedUpdater.createFeed(url, self.database) + feed = Feed.create(url) + feed.store(self.database) + FeedUpdater(self.database, self.preferences()).update(feed) def updateFeed(self, feed, changes): for key in changes.keys(): @@ -96,7 +98,7 @@ allFeeds = Feed.all(self.database) for feed in allFeeds: if feed.needsUpdate(): - FeedUpdater(feed, self.database).update() + FeedUpdater(self.database, self.preferences()).update(feed) def expireFeedEntries(self): print("Expiring feeds is not yet implemented")
--- a/backend/couchdb/Feed.py Mon Aug 29 04:12:01 2011 +0200 +++ b/backend/couchdb/Feed.py Sat Sep 03 04:12:35 2011 +0200 @@ -16,7 +16,7 @@ return Feed.view(database, "feedtest/feeds") @staticmethod - def create(url, title): + def create(url, title=None): feed = Feed() feed.rss_url = url feed.title = title
--- a/backend/couchdb/FeedUpdater.py Mon Aug 29 04:12:01 2011 +0200 +++ b/backend/couchdb/FeedUpdater.py Sat Sep 03 04:12:35 2011 +0200 @@ -1,26 +1,13 @@ -from Feed import Feed from FeedEntry import FeedEntry from backend.AbstractFeedUpdater import AbstractFeedUpdater import logging -import feedparser log = logging.getLogger("FeedUpdater") class FeedUpdater(AbstractFeedUpdater): - @staticmethod - def createFeed(url, database): - feedDict = feedparser.parse(url) - if feedDict.has_key("title"): - title = feedDict["feed"].title - else: - title = url - feed = Feed.create(url, title) - feed.store(database) - FeedUpdater(feed, database).update(feedDict) - - def __init__(self, feed, database): - AbstractFeedUpdater.__init__(self, feed) + def __init__(self, database, preferences): + AbstractFeedUpdater.__init__(self, preferences) self.database = database def _processEntry(self, entry): @@ -41,3 +28,9 @@ def _incrementFeedUpdateDate(self): self.feed.incrementNextUpdateDate() self.feed.store(self.database) + + def _setFeedTitle(self, feedDict): + oldTitle = self.feed.title + AbstractFeedUpdater._setFeedTitle(self, feedDict) + if oldTitle != self.feed.title: + self.feed.store(self.database)