Mercurial > hg > Feedworm

annotate migrate_couch_to_arango.py @ 256:f79be01821c4

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Arangodb backend, first version which barely works for reading
author Dirk Olmes <dirk@xanthippe.ping.de>
date Wed, 30 Jan 2019 07:11:10 +0100
parents b4c83e9b9c7a
children 75b81da8d7a5
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
255
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
1 #!/usr/bin/env python
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
2 # -*- coding: utf-8 -*-
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
3
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
4 import couchdb
256
f79be01821c4 Arangodb backend, first version which barely works for reading
Dirk Olmes <dirk@xanthippe.ping.de>
parents: 255
diff changeset
5 from datetime import datetime
255
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
6 import pyArango.connection
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
7
256
f79be01821c4 Arangodb backend, first version which barely works for reading
Dirk Olmes <dirk@xanthippe.ping.de>
parents: 255
diff changeset
8
255
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
9 def get_or_cretae_arango_collection(arango_db, collection_name):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
10 if arango_db.hasCollection(collection_name):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
11 return arango_db[collection_name]
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
12 else:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
13 return arango_db.createCollection(name=collection_name)
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
14
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
15 def migrate_feed(couch_doc):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
16 couch_id = couch_doc['_id']
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
17 if couch_id.startswith('_design'):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
18 return
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
19 try:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
20 if couch_doc['doctype'] == 'feed':
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
21 arango_doc = arango_feed.createDocument()
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
22 copy(couch_doc, arango_doc)
256
f79be01821c4 Arangodb backend, first version which barely works for reading
Dirk Olmes <dirk@xanthippe.ping.de>
parents: 255
diff changeset
23 convert_date(couch_doc, arango_doc, 'next_update')
255
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
24 arango_doc.save()
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
25 feed_mapping[couch_id] = arango_doc['_key']
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
26 except KeyError:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
27 print('**** migrate error ' + str(document))
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
28
256
f79be01821c4 Arangodb backend, first version which barely works for reading
Dirk Olmes <dirk@xanthippe.ping.de>
parents: 255
diff changeset
29 def convert_date(couch_doc, arango_doc, key):
f79be01821c4 Arangodb backend, first version which barely works for reading
Dirk Olmes <dirk@xanthippe.ping.de>
parents: 255
diff changeset
30 date_string = couch_doc[key]
f79be01821c4 Arangodb backend, first version which barely works for reading
Dirk Olmes <dirk@xanthippe.ping.de>
parents: 255
diff changeset
31 couch_date = datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ')
f79be01821c4 Arangodb backend, first version which barely works for reading
Dirk Olmes <dirk@xanthippe.ping.de>
parents: 255
diff changeset
32 # ignore the milliseconds - if they are printed through %f the number of
f79be01821c4 Arangodb backend, first version which barely works for reading
Dirk Olmes <dirk@xanthippe.ping.de>
parents: 255
diff changeset
33 # digits is not what arangodb expects
f79be01821c4 Arangodb backend, first version which barely works for reading
Dirk Olmes <dirk@xanthippe.ping.de>
parents: 255
diff changeset
34 date_string = couch_date.strftime('%Y-%m-%dT%H:%M:%S.000')
f79be01821c4 Arangodb backend, first version which barely works for reading
Dirk Olmes <dirk@xanthippe.ping.de>
parents: 255
diff changeset
35 arango_doc[key] = date_string
f79be01821c4 Arangodb backend, first version which barely works for reading
Dirk Olmes <dirk@xanthippe.ping.de>
parents: 255
diff changeset
36
255
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
37 def migrate_rest(document):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
38 if document['_id'].startswith('_design'):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
39 return
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
40 try:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
41 doctype = document['doctype']
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
42 if doctype == 'feed':
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
43 return
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
44 if doctype == 'feedEntry':
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
45 migrate_feed_entry(document)
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
46 elif doctype == 'preferences':
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
47 migrate_preferences(document)
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
48 else:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
49 print('how to migrate ' + document['_id'])
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
50 except KeyError:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
51 print('**** migrate error ' + str(document))
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
52
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
53 def migrate_feed_entry(couch_doc):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
54 arango_doc = arango_feed_entry.createDocument()
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
55 copy(couch_doc, arango_doc)
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
56 feed_id = arango_doc['feed']
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
57 feed_id = feed_mapping[feed_id]
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
58 arango_doc['feed'] = feed_id
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
59 arango_doc.save()
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
60
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
61 def migrate_preferences(couch_doc):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
62 arango_doc = arango_preferences.createDocument()
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
63 copy(couch_doc, arango_doc)
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
64 arango_doc.save()
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
65
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
66 def copy(couch_doc, arango_doc):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
67 for key in couch_doc:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
68 if not key.startswith('_'):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
69 arango_doc[key] = couch_doc[key]
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
70
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
71 if __name__ == '__main__':
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
72 couch_server = couchdb.Server('http://192.168.2.1:5984/')
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
73 couch_db = couch_server['feedworm']
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
74
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
75 arango_connection = pyArango.connection.Connection(arangoURL='http://xanthippe:8529', username='root', password='kahbHeotQDYL4R5o')
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
76 arango_db = arango_connection['feedworm']
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
77 arango_feed = get_or_cretae_arango_collection(arango_db, 'feed')
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
78 arango_feed_entry = get_or_cretae_arango_collection(arango_db, 'feed_entry')
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
79 arango_preferences = get_or_cretae_arango_collection(arango_db, 'preferences')
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
80
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
81 feed_mapping = {}
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
82
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
83 for id in couch_db:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
84 doc = couch_db[id]
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
85 migrate_feed(doc)
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
86
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
87 for id in couch_db:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
88 doc = couch_db[id]
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
89 migrate_rest(doc)