Created
January 29, 2010 21:46
-
-
Save TomK32/290175 to your computer and use it in GitHub Desktop.
Revisions
-
TomK32 revised this gist
Feb 3, 2010 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -105,7 +105,7 @@ def self.up # And now again all users and update their website_ids MongoStream::User.all.each do |m_user| old_website_ids = User.find(m_user.old_id).website_ids m_user.update_attribute(:website_ids, MongoStream::Website.all(:conditions => {:old_id => old_website_ids}).collect(&:id)) end # The best is to clean up and remove the old_ids via the mongo console, there for mongo 1.3+ -
TomK32 revised this gist
Feb 3, 2010 . 1 changed file with 4 additions and 3 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -40,8 +40,9 @@ class Page class User include MongoMapper::Document @collection_name = 'users' key :website_ids, Array has_many :sources, :class_name => "MongoStream::Source" has_many :websites, :class_name => "MongoStream::Website", :in => :website_ids has_many :photos, :class_name => "MongoStream::Photo" # has_and_belongs_to_many :websites end @@ -102,9 +103,9 @@ def self.up m_website.save end # And now again all users and update their website_ids MongoStream::User.all.each do |m_user| old_website_ids = User.find(m_user.old_id).website_ids m_user.website_ids = MongoStream::Website.all(:conditions => {:old_id => old_website_ids}).collect(&:id) end # The best is to clean up and remove the old_ids via the mongo console, there for mongo 1.3+ -
TomK32 renamed this gist
Feb 3, 2010 . 1 changed file with 0 additions and 0 deletions.There are no files selected for viewing
File renamed without changes. -
TomK32 revised this gist
Feb 3, 2010 . 1 changed file with 14 additions and 5 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -21,9 +21,9 @@ class Photo # has_and_belongs_to_many :albums end class Source include MongoMapper::Document @collection_name = 'sources' has_many :photos, :class_name => "MongoStream::Photo" # belongs_to :user end class Album @@ -42,6 +42,7 @@ class User @collection_name = 'users' has_many :sources, :class_name => "MongoStream::Source" has_many :websites, :class_name => "MongoStream::Website" has_many :photos, :class_name => "MongoStream::Photo" # has_and_belongs_to_many :websites end class Website @@ -68,16 +69,18 @@ def self.clean_attrs(object, unneeded_attributes = []) attributes end def self.up %w(MongoStream::User MongoStream::Website MongoStream::Photo MongoStream::Source).map{|klass| instance_eval("#{klass}.delete_all") rescue nil } ::User.all.each do |user| m_user = MongoStream::User.create!(clean_attrs(user)) user.sources.find(:all, :limit => 10).each do |source| m_source = MongoStream::Source.new(clean_attrs(source)) m_source[:user_id] = m_user.id source.photos.each do |photo| m_photo = MongoStream::Photo.create!(clean_attrs(photo)) m_photo[:tags] = photo.tag_list.to_a m_photo[:source_id] = m_source.id m_photo[:user_id] = m_source.user_id m_photo.save end m_source.save @@ -98,6 +101,12 @@ def self.up end m_website.save end # And now again all users and update their website_ids MongoStreamUser.all.each do |m_user| old_website_ids = User.find(m_user.old_id).website_ids m_user.website_ids = MongoStream::Website.find(:conditions => {:old_id => old_website_ids}).collect(&:id) end # The best is to clean up and remove the old_ids via the mongo console, there for mongo 1.3+ # db.photos.update({}, { $unset : { old_id : 1}}, false, true ) # db.websites.update({}, { $unset : { old_id : 1, 'albums.old_id': 1, 'pages.old_id': 1}}, false, true ) -
TomK32 revised this gist
Jan 31, 2010 . 1 changed file with 17 additions and 15 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -72,7 +72,7 @@ def self.up ::User.all.each do |user| m_user = MongoStream::User.create!(clean_attrs(user)) user.sources.all.each do |source| m_source = MongoStream::Source.new(clean_attrs(source)) source.photos.each do |photo| m_photo = MongoStream::Photo.create!(clean_attrs(photo)) @@ -83,24 +83,26 @@ def self.up m_source.save m_user.sources << m_source end # With those embedded documents, never forget to save the root element! m_user.save end Website.all.each do |website| m_website = MongoStream::Website.create(clean_attrs(website)) m_website.photos = MongoStream::Photo.all(:conditions => {:old_id => website.photo_ids}) m_website.user_ids = MongoStream::User.all(:conditions => {:old_id => website.user_ids}).collect(&:_id) website.albums.each do |album| m_website.albums << MongoStream::Album.new(clean_attrs(album, %w(website_id key_photo_id parent_id))) end website.pages.each do |page| m_website.pages << MongoStream::Page.new(clean_attrs(page, %w(website_id user_id parent_id))) end m_website.save end # The best is to clean up and remove the old_ids via the mongo console, there for mongo 1.3+ # db.photos.update({}, { $unset : { old_id : 1}}, false, true ) # db.websites.update({}, { $unset : { old_id : 1, 'albums.old_id': 1, 'pages.old_id': 1}}, false, true ) # db.users.update({}, { $unset : { old_id : 1}}, false, true ) end def self.down -
TomK32 revised this gist
Jan 31, 2010 . 1 changed file with 33 additions and 12 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -11,29 +11,38 @@ # Photo stays independed though I'm thinking about making copies into Album and Page # as this would allow the user to change e.g. title or tags in his photos # MongoStream is just some name as my app is called photostre.am module MongoStream class Photo include MongoMapper::Document @collection_name = 'photos' # belongs_to :source # has_and_belongs_to_many :websites # has_and_belongs_to_many :albums end class Source include MongoMapper::EmbeddedDocument key :photo_ids, Array has_many :photos, :class_name => "MongoStream::Photo", :in => :photo_ids # belongs_to :user end class Album include MongoMapper::EmbeddedDocument key :photo_ids, Array has_many :photos, :class_name => "MongoStream::Photo", :in => :photo_ids # belongs_to :website # belongs_to :key_photo, :class_name => 'Photo' end class Page include MongoMapper::EmbeddedDocument # belongs_to :website end class User include MongoMapper::Document @collection_name = 'users' has_many :sources, :class_name => "MongoStream::Source" has_many :websites, :class_name => "MongoStream::Website" # has_and_belongs_to_many :websites end class Website include MongoMapper::Document @@ -44,41 +53,53 @@ class Website has_many :pages, :class_name => "MongoStream::Page" has_many :photos, :class_name => "MongoStream::Photo", :in => :photo_ids has_many :users, :class_name => "MongoStream::User", :in => :user_ids # has_and_belongs_to_many :users end end class MigrateToMongodb < ActiveRecord::Migration require 'mongo_mapper' def self.clean_attrs(object, unneeded_attributes = []) unneeded_attributes << 'id' attributes = object.attributes.dup # we keep the old_id for now to copy the associations much easier attributes['old_id'] = attributes['id'] attributes.reject!{|k,v|unneeded_attributes.include?(k.to_s) || v.nil?} attributes end def self.up %w(MongoStream::User MongoStream::Website MongoStream::Photo).map{|klass| instance_eval("#{klass}.delete_all") rescue nil } ::User.all.each do |user| m_user = MongoStream::User.create!(clean_attrs(user)) user.sources.find(:all, :limit => 10).each do |source| m_source = MongoStream::Source.new(clean_attrs(source)) source.photos.each do |photo| m_photo = MongoStream::Photo.create!(clean_attrs(photo)) m_photo[:tags] = photo.tag_list.to_a m_photo[:source_id] = m_source.id m_photo.save end m_source.save m_user.sources << m_source end # With those embedded documents, never forget to save the root element! m_user.save user.websites.all.each do |website| m_website = MongoStream::Website.create(clean_attrs(website)) puts y website.photo_ids puts MongoStream::Photo.all(:conditions => {:old_id => website.photo_ids}) m_website.photos = MongoStream::Photo.all(:conditions => {:old_id => website.photo_ids}) m_website.user_ids = MongoStream::User.all(:conditions => {:old_id => website.user_ids}).collect(&:_id) website.albums.each do |album| m_website.albums << MongoStream::Album.new(clean_attrs(album, %w(website_id key_photo_id))) end website.pages.each do |page| m_website.pages << MongoStream::Page.new(clean_attrs(page)) end m_website.save end end end -
TomK32 revised this gist
Jan 31, 2010 . 1 changed file with 37 additions and 17 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -5,59 +5,79 @@ # We rely on models still being ActiveRecord::Base, I bet you can figure out how the look like. # And have the newer MongoDB ones here in a module, painful as we have to set the collection_name # Don't put a +timestamps!+ into your MongoMapper models yet because this would change the updated_at if existing # As you see in the MongoDB models, a few loose their indepence, e.g. Source as I # plan to add other sources besides flickr, or Page and Album which only make sense in # their parent Website # Photo stays independed though I'm thinking about making copies into Album and Page # as this would allow the user to change e.g. title or tags in his photos module MongoStream class Photo include MongoMapper::Document @collection_name = 'photos' end class Source include MongoMapper::EmbeddedDocument key :photo_ids, Array has_many :photos, :class_name => "MongoStream::Photo", :in => :photo_ids end class Album include MongoMapper::EmbeddedDocument key :photo_ids, Array has_many :photos, :class_name => "MongoStream::Photo", :in => :photo_ids end class Page include MongoMapper::EmbeddedDocument end class User include MongoMapper::Document @collection_name = 'users' has_many :sources, :class_name => "MongoStream::Source" has_many :websites, :class_name => "MongoStream::Website" end class Website include MongoMapper::Document @collection_name = 'websites' key :photo_ids, Array key :user_ids, Array has_many :albums, :class_name => "MongoStream::Album" has_many :pages, :class_name => "MongoStream::Page" has_many :photos, :class_name => "MongoStream::Photo", :in => :photo_ids has_many :users, :class_name => "MongoStream::User", :in => :user_ids end end class MigrateToMongodb < ActiveRecord::Migration require 'mongo_mapper' def self.clean_attrs(object) attributes = object.attribtes.dup # we keep the old_id for now to copy the associations much easier attributes[:old_id] = attributes[:id] attributes.reject{|k,v|k == :id || v.nil?} end def self.up %w(MongoStream::User MongoStream::Website MongoStream::Photo).map{|klass| instance_eval("#{klass}.delete_all") rescue nil } ::User.all.each do |user| m_user = MongoStream::User.create!(clean_attrs(user)) user.sources.all.each do |source| m_source = MongoStream::Source.new(clean_attrs(source)) source.photos.each do |photo| m_photo = MongoStream::Photo.create!(clean_attrs(photo)) m_photo[:tags] = photo.tag_list.to_a m_photo[:source_id] = m_source.id m_photo.save photo.websites.each do |w| m_website = MongoStream::Website.first(:domain => w.domain) m_website.photos << m_photo end end m_source.save m_user.sources << m_source end user.websites.all.each do |website| m_website = Website.create(clean_attrs(website.attributes)) end # With those embedded documents, never forget to save the root element! m_user.save end end -
TomK32 revised this gist
Jan 31, 2010 . 1 changed file with 0 additions and 2 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -25,14 +25,12 @@ class Page class User @collection_name = 'users' include MongoMapper::Document has_many :sources, :class_name => "MongoStream::Source" has_many :websites, :class_name => "MongoStream::Website" end class Website @collection_name = 'websites' include MongoMapper::Document has_many :pages, :class_name => "MongoStream::Page" has_many :users, :class_name => "MongoStream::User" has_many :photos, :class_name => "MongoStream::Photo" -
TomK32 revised this gist
Jan 29, 2010 . 1 changed file with 13 additions and 6 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,6 +1,12 @@ # The basic idea is that we take the existing data via ActiveRecord # and create new documents in MongoDB using MongoMapper. # This method is necessary as we want to keep all the associations of existing dataset # and by the way, clean up empty columns # We rely on models still being ActiveRecord::Base, I bet you can figure out how the look like. # And have the newer MongoDB ones here in a module, painful as we have to set the collection_name # Don't put a +timestamps!+ into your MongoMapper models yet because this would change the updated_at if existing module MongoStream class Photo @collection_name = 'photos' include MongoMapper::Document @@ -40,18 +46,19 @@ def self.clean_attrs(attributes) attributes.reject{|k,v|k=="id" || v.nil?} end def self.up %w(MongoStream::User MongoStream::Website MongoStream::Photo).map{|klass| instance_eval("#{klass}.delete_all") rescue nil } ::User.all.each do |user| m_user = MongoStream::User.create!(clean_attrs(user.attributes)) user.sources.all.each do |source| m_source = MongoStream::Source.new(clean_attrs(source.attributes)) source.photos.each do |photo| p = MongoStream::Photo.create!(clean_attrs(photo.attributes)) p[:tags] = photo.tag_list.to_a p[:source_id] = m_source.id p.save end m_source.save m_user.sources << m_source end m_user.save end -
TomK32 revised this gist
Jan 29, 2010 . 1 changed file with 19 additions and 14 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,14 +1,9 @@ module MongoStream # we rely on models still being ActiveRecord::Base # and have the newer MongoDB ones here. class Photo @collection_name = 'photos' include MongoMapper::Document end class Source include MongoMapper::EmbeddedDocument @@ -24,12 +19,14 @@ class Page class User @collection_name = 'users' include MongoMapper::Document key :website_id has_many :sources, :class_name => "MongoStream::Source" has_many :websites, :class_name => "MongoStream::Website" end class Website @collection_name = 'websites' include MongoMapper::Document key :website_id has_many :pages, :class_name => "MongoStream::Page" has_many :users, :class_name => "MongoStream::User" has_many :photos, :class_name => "MongoStream::Photo" @@ -39,14 +36,22 @@ class Website class MigrateToMongodb < ActiveRecord::Migration require 'mongo_mapper' def self.clean_attrs(attributes) attributes.reject{|k,v|k=="id" || v.nil?} end def self.up %w(MongoStream::User MongoStream::Website).map{|klass| instance_eval("#{klass}.delete_all") rescue nil } ::User.all.each do |user| m_user = MongoStream::User.create!(clean_attrs(user.attributes)) user.sources.all.each do |source| m_user.sources << MongoStream::Source.new(clean_attrs(source.attributes)) source.photos.each do |photo| p = MongoStream::Photo.create!(clean_attrs(photo.attributes)) p[:tags] = photo.tag_list.to_a p.save source.photo_ids << p.id end source.save end m_user.save end -
TomK32 revised this gist
Jan 29, 2010 . 1 changed file with 5 additions and 2 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,7 +1,10 @@ # The basic idea is that we take the existing data via ActiveRecord # and create new documents in MongoDB using MongoMapper. # This method is necessary as we want to keep all the associations of existing dataset # and by the way, clean up empty columns # We rely on models still being ActiveRecord::Base, I bet you can figure out how the look like. # And have the newer MongoDB ones here in a module, painful as we have to set the collection_name # Don't put a +timestamps!+ into your MongoMapper models yet because this would change the updated_at if existing module MongoStream class Photo -
TomK32 revised this gist
Jan 29, 2010 . 1 changed file with 2 additions and 2 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,5 +1,5 @@ # The basic idea is that we take the existing data via ActiveRecord and create new documents in MongoDB using MongoMapper. # This method is necessary as we want to keep all the associations of existing dataset and by the way, clean up empty columns # We rely on models still being ActiveRecord::Base, I bet you can figure out how the look like. # And have the newer MongoDB ones here in a module, painful as we have to set the collection_name -
TomK32 renamed this gist
Jan 29, 2010 . 1 changed file with 5 additions and 6 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,6 +1,9 @@ # The basic idea is that we take the existing data via ActiveRecord and create new documents in MongoDB using # MongoMapper. This method is necessary as we want to keep all the associations of existing dataset. # We rely on models still being ActiveRecord::Base, I bet you can figure out how the look like. # And have the newer MongoDB ones here in a module, painful as we have to set the collection_name module MongoStream class Photo include MongoMapper::EmbeddedDocument end @@ -18,14 +21,12 @@ class Page class User @collection_name = 'users' include MongoMapper::Document has_many :sources, :class_name => "MongoStream::Source" has_many :websites, :class_name => "MongoStream::Website" end class Website @collection_name = 'websites' include MongoMapper::Document has_many :pages, :class_name => "MongoStream::Page" has_many :users, :class_name => "MongoStream::User" has_many :photos, :class_name => "MongoStream::Photo" @@ -36,8 +37,6 @@ class Website class MigrateToMongodb < ActiveRecord::Migration require 'mongo_mapper' def self.up %w(MongoStream::User MongoStream::Website).map{|klass| instance_eval("#{klass}.delete_all") rescue nil } -
TomK32 created this gist
Jan 29, 2010 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,55 @@ module MongoStream # we rely on models still being ActiveRecord::Base # and have the newer MongoDB ones here. class Photo include MongoMapper::EmbeddedDocument end class Source include MongoMapper::EmbeddedDocument end class Album include MongoMapper::EmbeddedDocument has_many :photos, :class_name => "MongoStream::Photo" end class Page include MongoMapper::EmbeddedDocument has_many :photos, :class_name => "MongoStream::Photo" end class User @collection_name = 'users' include MongoMapper::Document key :website_id has_many :sources, :class_name => "MongoStream::Source" has_many :websites, :class_name => "MongoStream::Website" end class Website @collection_name = 'websites' include MongoMapper::Document key :website_id has_many :pages, :class_name => "MongoStream::Page" has_many :users, :class_name => "MongoStream::User" has_many :photos, :class_name => "MongoStream::Photo" has_many :albums, :class_name => "MongoStream::Album" end end class MigrateToMongodb < ActiveRecord::Migration require 'mongo_mapper' def self.up %w(MongoStream::User MongoStream::Website).map{|klass| instance_eval("#{klass}.delete_all") rescue nil } ::User.all.each do |user| m_user = MongoStream::User.create!(user.attributes.reject{|k,v|k=="id" || v.nil?}) user.sources.all.each do |s| m_user.sources << MongoStream::Source.new(s.attributes.reject{|k,v|%w(id website_id).include?(k) || v.nil?}) end m_user.save end end def self.down end end