# HG changeset patch # User Dirk Olmes # Date 1548828670 -3600 # Node ID f79be01821c4846eed354433bea8b8a0a9c7de5d # Parent b4c83e9b9c7a9d73c7ce60b50e0a16506852d98f Arangodb backend, first version which barely works for reading diff -r b4c83e9b9c7a -r f79be01821c4 BackendFactory.py --- a/BackendFactory.py Thu Nov 29 18:46:21 2018 +0100 +++ b/BackendFactory.py Wed Jan 30 07:11:10 2019 +0100 @@ -1,13 +1,14 @@ # -*- coding: utf-8 -*- import argparse +ARANGODB_BACKEND = "arangodb" +COUCHDB_BACKEND = "couchdb" SQLALCHEMY_BACKEND = "sqlalchemy" -COUCHDB_BACKEND = "couchdb" def _parseArguments(): parser = argparse.ArgumentParser() - parser.add_argument("--backend", nargs="?", choices=[SQLALCHEMY_BACKEND, COUCHDB_BACKEND], - required=True, help="Specify the backend to use: either sqlalchemy or couchdb") + parser.add_argument("--backend", nargs="?", choices=[ARANGODB_BACKEND, COUCHDB_BACKEND, SQLALCHEMY_BACKEND], + required=True, help="Specify the backend to use: either arangodb, couchdb or sqlalchemy") return parser.parse_known_args() def createBackend(): @@ -19,5 +20,8 @@ elif backend == COUCHDB_BACKEND: from backend.couchdb.CouchDbBackend import CouchDbBackend return CouchDbBackend() + elif backend == ARANGODB_BACKEND: + from backend.arangodb.ArangoBackend import ArangoBackend + return ArangoBackend() else: raise Exception("no backend configured") diff -r b4c83e9b9c7a -r f79be01821c4 backend/AbstractBackend.py --- a/backend/AbstractBackend.py Thu Nov 29 18:46:21 2018 +0100 +++ b/backend/AbstractBackend.py Wed Jan 30 07:11:10 2019 +0100 @@ -20,6 +20,12 @@ # # handling of feeds # + def getFeeds(self): + if self.preferences().showOnlyUnreadFeeds(): + self.feeds = self.getUnreadFeeds() + else: + self.feeds = self.getAllFeeds() + return self.feeds def selectFeed(self, index): self.selectedFeed = self.feeds[index] diff -r b4c83e9b9c7a -r f79be01821c4 backend/AbstractPreferences.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/backend/AbstractPreferences.py Wed Jan 30 07:11:10 2019 +0100 @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- + +class AbstractPreferences(object): + HIDE_READ_FEED_ENTRIES = "hideReadFeedEntries" + PROXY_HOST = "proxyHost" + SHOW_ONLY_UNREAD_FEEDS = "showOnlyUnreadFeeds" + START_MAXIMIZED = "startMaximized" + USE_PROXY = "useProxy" + + def isProxyConfigured(self): + return self.proxyHost() is not None diff -r b4c83e9b9c7a -r f79be01821c4 backend/arangodb/ArangoBackend.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/backend/arangodb/ArangoBackend.py Wed Jan 30 07:11:10 2019 +0100 @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +from argparse import ArgumentParser +from ArangoDb import ArangoDb +from backend.AbstractBackend import AbstractBackend +from Feed import Feed +from FeedEntry import FeedEntry +from Preferences import Preferences +from pyArango.connection import Connection + +""" +Backend that uses ArangoDB for persistence +""" +class ArangoBackend(AbstractBackend): + def __init__(self): + super(ArangoBackend, self).__init__() + args = self._parse_arguments() + connection = Connection(arangoURL=args.dburl, username=args.user, password=args.password) + self.database = ArangoDb(connection[args.dbname]) + self.prefs = None + + def _parse_arguments(self): + parser = ArgumentParser() + parser.add_argument('--dburl', nargs='?', help='URL of the database', default='http://127.0.0.1:8529') + parser.add_argument('--dbname', nargs='?', help='name of the database', default='feedworm') + parser.add_argument('--user', nargs='?', help='username for authenticating the database connection', required=True) + parser.add_argument('--password', nargs='?', help='password for authenticating the database connection', required=True) + return parser.parse_known_args()[0] + + def preferences(self): + if self.prefs is None: + self.prefs = Preferences(self.database) + return self.prefs + + def getUnreadFeeds(self): + return Feed.get_unread(self.database) + + def _retrieveEntriesForSelectedFeed(self, hideReadEntries): + base_query = """ + FOR feed_entry_doc in feed_entry + FILTER feed_entry_doc.feed == @feed_key""" + if hideReadEntries: + query = base_query + " AND feed_entry_doc.read == false " + query = query + " RETURN feed_entry_doc" + bind_vars = { 'feed_key': self.selectedFeed._key } + results = self.database.AQLQuery(query, bind_vars=bind_vars) + return [FeedEntry(doc) for doc in results] + + def _markSelectedFeedEntryRead(self): + self.selectedFeedEntry.markRead() + + def updateAllFeeds(self): + for feed in Feed.all_pending_update(self.database): + print('updating ' + feed.title) diff -r b4c83e9b9c7a -r f79be01821c4 backend/arangodb/ArangoDb.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/backend/arangodb/ArangoDb.py Wed Jan 30 07:11:10 2019 +0100 @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +class ArangoDb(object): + def __init__(self, database): + super(ArangoDb, self).__init__() + self.database = database + + def get_or_create_collection(self, collection_name): + if self.database.hasCollection(collection_name): + return self.database[collection_name] + else: + return self.database.createCollection(name=collection_name) + + def AQLQuery(self, query, bind_vars={}): + return self.database.AQLQuery(query, bindVars=bind_vars) \ No newline at end of file diff -r b4c83e9b9c7a -r f79be01821c4 backend/arangodb/Feed.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/backend/arangodb/Feed.py Wed Jan 30 07:11:10 2019 +0100 @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +from datetime import datetime + +class Feed(object): + def __init__(self, document): + super(Feed, self).__init__() + self.document = document + + def __getattr__(self, attribute): + return self.document[attribute] + + @staticmethod + def get_unread(database): + query = """ + FOR feed_entry_doc IN feed_entry + FOR feed_doc IN feed + FILTER feed_entry_doc.read == false + AND feed_entry_doc.feed == feed_doc._key + RETURN DISTINCT feed_doc""" + results = database.AQLQuery(query) + return [Feed(doc) for doc in results] + + @staticmethod + def all_pending_update(database): + query = """ + FOR feed_doc IN feed + FILTER DATE_ISO8601(DATE_NOW()) > feed_doc.next_update + RETURN feed_doc + """ + results = database.AQLQuery(query) + return [Feed(doc) for doc in results] diff -r b4c83e9b9c7a -r f79be01821c4 backend/arangodb/FeedEntry.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/backend/arangodb/FeedEntry.py Wed Jan 30 07:11:10 2019 +0100 @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from datetime import datetime + +class FeedEntry(object): + def __init__(self, document): + super(FeedEntry, self).__init__() + self.document = document + + def __getattr__(self, attribute): + if attribute == 'updated': + return self._parse_updated() + return self.document[attribute] + + def _parse_updated(self): + value = self.document['updated'] + return datetime(*value) + + def markRead(self): + self.document['read'] = True + self.document.patch() diff -r b4c83e9b9c7a -r f79be01821c4 backend/arangodb/Preferences.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/backend/arangodb/Preferences.py Wed Jan 30 07:11:10 2019 +0100 @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +from ..AbstractPreferences import AbstractPreferences + +class Preferences(AbstractPreferences): + def __init__(self, database): + super(Preferences, self).__init__() + collection = database.get_or_create_collection('preferences') + self.doc = collection.fetchAll(limit=1)[0] + + def hideReadFeedEntries(self): + return self.doc[self.HIDE_READ_FEED_ENTRIES] + + def proxyHost(self): + return self.doc[self.PROXY_HOST] + + def showOnlyUnreadFeeds(self): + return self.doc[self.SHOW_ONLY_UNREAD_FEEDS] + + def startMaximized(self): + return self.doc[self.START_MAXIMIZED] + + def useProxy(self): + return self.doc[self.USE_PROXY] diff -r b4c83e9b9c7a -r f79be01821c4 backend/arangodb/__init__.py diff -r b4c83e9b9c7a -r f79be01821c4 backend/couchdb/CouchDbBackend.py --- a/backend/couchdb/CouchDbBackend.py Thu Nov 29 18:46:21 2018 +0100 +++ b/backend/couchdb/CouchDbBackend.py Wed Jan 30 07:11:10 2019 +0100 @@ -34,17 +34,7 @@ # # handling of feeds # - - def getFeeds(self): - if self.preferences().showOnlyUnreadFeeds(): - self.feeds = self._getUnreadFeeds() - else: - # make sure that the results are actually fetched into memory, otherwise we'll pass - # a ViewResults instance around which is not what we want - self.feeds = list(Feed.all(self.database)) - return self.feeds - - def _getUnreadFeeds(self): + def getUnreadFeeds(self): viewResults = self.database.view(CouchDb.feedsWithUnreadEntries(), group=True) feedsWithUnreadEntries = [] for row in viewResults: @@ -52,6 +42,11 @@ feedsWithUnreadEntries.append(feed) return feedsWithUnreadEntries + def getAllFeeds(self): + # make sure that the results are actually fetched into memory, otherwise we'll pass + # a ViewResults instance around which is not what we want + return list(Feed.all(self.database)) + def _retrieveEntriesForSelectedFeed(self, hideReadEntries): viewResults = FeedEntry.entriesForFeed(self.selectedFeed, self.database) if hideReadEntries: diff -r b4c83e9b9c7a -r f79be01821c4 backend/couchdb/Preferences.py --- a/backend/couchdb/Preferences.py Thu Nov 29 18:46:21 2018 +0100 +++ b/backend/couchdb/Preferences.py Wed Jan 30 07:11:10 2019 +0100 @@ -1,15 +1,11 @@ # -*- coding: utf-8 -*- +from ..AbstractPreferences import AbstractPreferences import CouchDb DAYS_TO_KEEP_FEED_ENTRIES = "daysToKeepFeedEntries" -HIDE_READ_FEED_ENTRIES = "hideReadFeedEntries" -PROXY_HOST = "proxyHost" PROXY_PORT = "proxyPort" -SHOW_ONLY_UNREAD_FEEDS = "showOnlyUnreadFeeds" -START_MAXIMIZED = "startMaximized" -USE_PROXY = "useProxy" -class Preferences(object): +class Preferences(AbstractPreferences): def __init__(self, database): self.database = database self._initDocument() @@ -35,14 +31,11 @@ self.document[key] = value self.documentIsDirty = True - def isProxyConfigured(self): - return self.proxyHost() is not None - def proxyHost(self): - return self._documentValue(PROXY_HOST) + return self._documentValue(self.PROXY_HOST) def useProxy(self): - return self._documentValue(USE_PROXY, True) + return self._documentValue(self.USE_PROXY, True) def setUseProxy(self, value): self._setDocumentValue(USE_PROXY, value) @@ -67,22 +60,22 @@ self._setDocumentValue(PROXY_PORT, port) def showOnlyUnreadFeeds(self): - return self._documentValue(SHOW_ONLY_UNREAD_FEEDS, False) + return self._documentValue(self.SHOW_ONLY_UNREAD_FEEDS, False) def setShowOnlyUnreadFeeds(self, flag): - self._setDocumentValue(SHOW_ONLY_UNREAD_FEEDS, flag) + self._setDocumentValue(self.SHOW_ONLY_UNREAD_FEEDS, flag) def startMaximized(self): - return self._documentValue(START_MAXIMIZED, False) + return self._documentValue(self.START_MAXIMIZED, False) def setStartMaximized(self, flag): self._setDocumentValue(START_MAXIMIZED, flag) def hideReadFeedEntries(self): - return self._documentValue(HIDE_READ_FEED_ENTRIES, False) + return self._documentValue(self.HIDE_READ_FEED_ENTRIES, False) def setHideReadFeedEntries(self, flag): - self._setDocumentValue(HIDE_READ_FEED_ENTRIES, flag) + self._setDocumentValue(self.HIDE_READ_FEED_ENTRIES, flag) def daysToKeepFeedEntries(self): value = self._documentValue(DAYS_TO_KEEP_FEED_ENTRIES, 90) diff -r b4c83e9b9c7a -r f79be01821c4 migrate_couch_to_arango.py --- a/migrate_couch_to_arango.py Thu Nov 29 18:46:21 2018 +0100 +++ b/migrate_couch_to_arango.py Wed Jan 30 07:11:10 2019 +0100 @@ -2,8 +2,10 @@ # -*- coding: utf-8 -*- import couchdb +from datetime import datetime import pyArango.connection + def get_or_cretae_arango_collection(arango_db, collection_name): if arango_db.hasCollection(collection_name): return arango_db[collection_name] @@ -18,11 +20,20 @@ if couch_doc['doctype'] == 'feed': arango_doc = arango_feed.createDocument() copy(couch_doc, arango_doc) + convert_date(couch_doc, arango_doc, 'next_update') arango_doc.save() feed_mapping[couch_id] = arango_doc['_key'] except KeyError: print('**** migrate error ' + str(document)) +def convert_date(couch_doc, arango_doc, key): + date_string = couch_doc[key] + couch_date = datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ') + # ignore the milliseconds - if they are printed through %f the number of + # digits is not what arangodb expects + date_string = couch_date.strftime('%Y-%m-%dT%H:%M:%S.000') + arango_doc[key] = date_string + def migrate_rest(document): if document['_id'].startswith('_design'): return