Mercurial > hg > Feedworm

annotate migrate_couch_to_arango.py @ 255:b4c83e9b9c7a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
migration from couchdb to arangodb
author Dirk Olmes <dirk@xanthippe.ping.de>
date Thu, 29 Nov 2018 18:46:21 +0100
parents
children f79be01821c4
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
255
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
1 #!/usr/bin/env python
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
2 # -*- coding: utf-8 -*-
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
3
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
4 import couchdb
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
5 import pyArango.connection
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
6
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
7 def get_or_cretae_arango_collection(arango_db, collection_name):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
8 if arango_db.hasCollection(collection_name):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
9 return arango_db[collection_name]
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
10 else:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
11 return arango_db.createCollection(name=collection_name)
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
12
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
13 def migrate_feed(couch_doc):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
14 couch_id = couch_doc['_id']
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
15 if couch_id.startswith('_design'):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
16 return
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
17 try:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
18 if couch_doc['doctype'] == 'feed':
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
19 arango_doc = arango_feed.createDocument()
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
20 copy(couch_doc, arango_doc)
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
21 arango_doc.save()
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
22 feed_mapping[couch_id] = arango_doc['_key']
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
23 except KeyError:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
24 print('**** migrate error ' + str(document))
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
25
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
26 def migrate_rest(document):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
27 if document['_id'].startswith('_design'):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
28 return
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
29 try:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
30 doctype = document['doctype']
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
31 if doctype == 'feed':
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
32 return
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
33 if doctype == 'feedEntry':
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
34 migrate_feed_entry(document)
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
35 elif doctype == 'preferences':
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
36 migrate_preferences(document)
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
37 else:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
38 print('how to migrate ' + document['_id'])
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
39 except KeyError:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
40 print('**** migrate error ' + str(document))
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
41
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
42 def migrate_feed_entry(couch_doc):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
43 arango_doc = arango_feed_entry.createDocument()
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
44 copy(couch_doc, arango_doc)
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
45 feed_id = arango_doc['feed']
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
46 feed_id = feed_mapping[feed_id]
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
47 arango_doc['feed'] = feed_id
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
48 arango_doc.save()
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
49
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
50 def migrate_preferences(couch_doc):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
51 arango_doc = arango_preferences.createDocument()
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
52 copy(couch_doc, arango_doc)
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
53 arango_doc.save()
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
54
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
55 def copy(couch_doc, arango_doc):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
56 for key in couch_doc:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
57 if not key.startswith('_'):
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
58 arango_doc[key] = couch_doc[key]
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
59
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
60 if __name__ == '__main__':
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
61 couch_server = couchdb.Server('http://192.168.2.1:5984/')
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
62 couch_db = couch_server['feedworm']
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
63
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
64 arango_connection = pyArango.connection.Connection(arangoURL='http://xanthippe:8529', username='root', password='kahbHeotQDYL4R5o')
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
65 arango_db = arango_connection['feedworm']
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
66 arango_feed = get_or_cretae_arango_collection(arango_db, 'feed')
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
67 arango_feed_entry = get_or_cretae_arango_collection(arango_db, 'feed_entry')
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
68 arango_preferences = get_or_cretae_arango_collection(arango_db, 'preferences')
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
69
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
70 feed_mapping = {}
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
71
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
72 for id in couch_db:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
73 doc = couch_db[id]
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
74 migrate_feed(doc)
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
75
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
76 for id in couch_db:
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
77 doc = couch_db[id]
b4c83e9b9c7a migration from couchdb to arangodb
Dirk Olmes <dirk@xanthippe.ping.de>
parents:
diff changeset
78 migrate_rest(doc)