--- /dev/null
+= YamlDb
+
+YamlDb is a database-independent format for dumping and restoring data. It complements the the database-independent schema format found in db/schema.rb. The data is saved into db/data.yml.
+
+This can be used as a replacement for mysqldump or pg_dump, but only for the databases typically used by Rails apps. Users, permissions, schemas, triggers, and other advanced database features are not supported - by design.
+
+Any database that has an ActiveRecord adapter should work.
+
+== Usage
+
+rake db:data:dump -> Dump contents of Rails database to db/data.yml
+rake db:data:load -> Load contents of db/data.yml into the database
+
+Further, there are tasks db:dump and db:load which do the entire database (the equivalent of running db:schema:dump followed by db:data:load).
+
+== Examples
+
+One common use would be to switch your data from one database backend to another. For example, let's say you wanted to switch from SQLite to MySQL. You might execute the following steps:
+
+1. rake db:dump
+
+2. Edit config/database.yml and change your adapter to mysql, set up database params
+
+3. mysqladmin create [database name]
+
+4. rake db:load
+
+== Credits
+
+Created by Orion Henry and Adam Wiggins. Major updates by Ricardo Chimal, Jr.
+
+Patches contributed by Michael Irwin, Tom Locke, and Tim Galeckas.
+
+Send questions, feedback, or patches to the Heroku mailing list: http://groups.google.com/group/heroku
+
--- /dev/null
+require 'rake'
+require 'spec/rake/spectask'
+
+desc "Run all specs"
+Spec::Rake::SpecTask.new('spec') do |t|
+ t.spec_files = FileList['spec/*_spec.rb']
+end
+
+task :default => :spec
+
--- /dev/null
+author: Orion Henry and Adam Wiggins of Heroku
+summary: Dumps and loads a database-independent data dump format in db/data.yml.
+homepage: http://opensource.heroku.com/
+license: MIT
+rails_version: 1.2+
--- /dev/null
+require 'yaml_db'
--- /dev/null
+require 'rubygems'
+require 'yaml'
+require 'active_record'
+
+
+module YamlDb
+ def self.dump(filename)
+ disable_logger
+ YamlDb::Dump.dump(File.new(filename, "w"))
+ reenable_logger
+ end
+
+ def self.load(filename)
+ disable_logger
+ YamlDb::Load.load(File.new(filename, "r"))
+ reenable_logger
+ end
+
+ def self.disable_logger
+ @@old_logger = ActiveRecord::Base.logger
+ ActiveRecord::Base.logger = nil
+ end
+
+ def self.reenable_logger
+ ActiveRecord::Base.logger = @@old_logger
+ end
+end
+
+
+module YamlDb::Utils
+ def self.chunk_records(records)
+ yaml = [ records ].to_yaml
+ yaml.sub!("--- \n", "")
+ yaml.sub!('- - -', ' - -')
+ yaml
+ end
+
+ def self.unhash(hash, keys)
+ keys.map { |key| hash[key] }
+ end
+
+ def self.unhash_records(records, keys)
+ records.each_with_index do |record, index|
+ records[index] = unhash(record, keys)
+ end
+
+ records
+ end
+
+ def self.convert_booleans(records, columns)
+ records.each do |record|
+ columns.each do |column|
+ next if is_boolean(record[column])
+ record[column] = (record[column] == 't' or record[column] == '1')
+ end
+ end
+ records
+ end
+
+ def self.boolean_columns(table)
+ columns = ActiveRecord::Base.connection.columns(table).reject { |c| c.type != :boolean }
+ columns.map { |c| c.name }
+ end
+
+ def self.is_boolean(value)
+ value.kind_of?(TrueClass) or value.kind_of?(FalseClass)
+ end
+
+ def self.quote_table(table)
+ ActiveRecord::Base.connection.quote_table_name(table)
+ end
+end
+
+
+module YamlDb::Dump
+ def self.dump(io)
+ tables.each do |table|
+ dump_table(io, table)
+ end
+ end
+
+ def self.tables
+ ActiveRecord::Base.connection.tables.reject { |table| ['schema_info', 'schema_migrations'].include?(table) }
+ end
+
+ def self.dump_table(io, table)
+ return if table_record_count(table).zero?
+
+ dump_table_columns(io, table)
+ dump_table_records(io, table)
+ end
+
+ def self.dump_table_columns(io, table)
+ io.write("\n")
+ io.write({ table => { 'columns' => table_column_names(table) } }.to_yaml)
+ end
+
+ def self.dump_table_records(io, table)
+ table_record_header(io)
+
+ column_names = table_column_names(table)
+
+ each_table_page(table) do |records|
+ rows = YamlDb::Utils.unhash_records(records, column_names)
+ io.write(YamlDb::Utils.chunk_records(records))
+ end
+ end
+
+ def self.table_record_header(io)
+ io.write(" records: \n")
+ end
+
+ def self.table_column_names(table)
+ ActiveRecord::Base.connection.columns(table).map { |c| c.name }
+ end
+
+ def self.each_table_page(table, records_per_page=1000)
+ total_count = table_record_count(table)
+ pages = (total_count.to_f / records_per_page).ceil - 1
+ id = table_column_names(table).first
+ boolean_columns = YamlDb::Utils.boolean_columns(table)
+ quoted_table_name = YamlDb::Utils.quote_table(table)
+
+ (0..pages).to_a.each do |page|
+ sql = ActiveRecord::Base.connection.add_limit_offset!("SELECT * FROM #{quoted_table_name} ORDER BY #{id}",
+ :limit => records_per_page, :offset => records_per_page * page
+ )
+ records = ActiveRecord::Base.connection.select_all(sql)
+ records = YamlDb::Utils.convert_booleans(records, boolean_columns)
+ yield records
+ end
+ end
+
+ def self.table_record_count(table)
+ ActiveRecord::Base.connection.select_one("SELECT COUNT(*) FROM #{YamlDb::Utils.quote_table(table)}").values.first.to_i
+ end
+end
+
+
+module YamlDb::Load
+ def self.load(io)
+ ActiveRecord::Base.connection.transaction do
+ YAML.load_documents(io) do |ydoc|
+ ydoc.keys.each do |table_name|
+ next if ydoc[table_name].nil?
+ load_table(table_name, ydoc[table_name])
+ end
+ end
+ end
+ end
+
+ def self.truncate_table(table)
+ begin
+ ActiveRecord::Base.connection.execute("TRUNCATE #{YamlDb::Utils.quote_table(table)}")
+ rescue Exception
+ ActiveRecord::Base.connection.execute("DELETE FROM #{YamlDb::Utils.quote_table(table)}")
+ end
+ end
+
+ def self.load_table(table, data)
+ column_names = data['columns']
+ truncate_table(table)
+ load_records(table, column_names, data['records'])
+ reset_pk_sequence!(table)
+ end
+
+ def self.load_records(table, column_names, records)
+ quoted_column_names = column_names.map { |column| ActiveRecord::Base.connection.quote_column_name(column) }.join(',')
+ quoted_table_name = YamlDb::Utils.quote_table(table)
+ records.each do |record|
+ ActiveRecord::Base.connection.execute("INSERT INTO #{quoted_table_name} (#{quoted_column_names}) VALUES (#{record.map { |r| ActiveRecord::Base.connection.quote(r) }.join(',')})")
+ end
+ end
+
+ def self.reset_pk_sequence!(table_name)
+ if ActiveRecord::Base.connection.respond_to?(:reset_pk_sequence!)
+ ActiveRecord::Base.connection.reset_pk_sequence!(table_name)
+ end
+ end
+end
--- /dev/null
+require 'rubygems'
+require 'spec'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../lib')
+require 'yaml_db'
+
+
--- /dev/null
+require File.dirname(__FILE__) + '/base'
+
+describe YamlDb::Dump do
+ before do
+ File.stub!(:new).with('dump.yml', 'w').and_return(StringIO.new)
+
+ ActiveRecord::Base = mock('ActiveRecord::Base', :null_object => true)
+ ActiveRecord::Base.connection = mock('connection')
+ ActiveRecord::Base.connection.stub!(:tables).and_return([ 'mytable', 'schema_info', 'schema_migrations' ])
+ ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('a',:name => 'a'), mock('b', :name => 'b') ])
+ ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"2"})
+ ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ])
+ YamlDb::Utils.stub!(:quote_table).with('mytable').and_return('mytable')
+ end
+
+ before(:each) do
+ @io = StringIO.new
+ end
+
+ it "should return a formatted string" do
+ YamlDb::Dump.table_record_header(@io)
+ @io.rewind
+ @io.read.should == " records: \n"
+ end
+
+ it "should return a list of column names" do
+ YamlDb::Dump.table_column_names('mytable').should == [ 'a', 'b' ]
+ end
+
+ it "should return a list of tables without the rails schema table" do
+ YamlDb::Dump.tables.should == ['mytable']
+ end
+
+ it "should return the total number of records in a table" do
+ YamlDb::Dump.table_record_count('mytable').should == 2
+ end
+
+ it "should return a yaml string that contains a table header and column names" do
+ YamlDb::Dump.stub!(:table_column_names).with('mytable').and_return([ 'a', 'b' ])
+ YamlDb::Dump.dump_table_columns(@io, 'mytable')
+ @io.rewind
+ @io.read.should == <<EOYAML
+
+---
+mytable:
+ columns:
+ - a
+ - b
+EOYAML
+ end
+
+ it "should return all records from the database and return them when there is only 1 page" do
+ YamlDb::Dump.each_table_page('mytable') do |records|
+ records.should == [ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ]
+ end
+ end
+
+ it "should paginate records from the database and return them" do
+ ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'a' => 1, 'b' => 2 } ], [ { 'a' => 3, 'b' => 4 } ])
+
+ records = [ ]
+ YamlDb::Dump.each_table_page('mytable', 1) do |page|
+ page.size.should == 1
+ records.concat(page)
+ end
+
+ records.should == [ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ]
+ end
+
+ it "should return dump the records for a table in yaml to a given io stream" do
+ YamlDb::Dump.dump_table_records(@io, 'mytable')
+ @io.rewind
+ @io.read.should == <<EOYAML
+ records:
+ - - 1
+ - 2
+ - - 3
+ - 4
+EOYAML
+ end
+
+ it "should dump a table's contents to yaml" do
+ YamlDb::Dump.should_receive(:dump_table_columns)
+ YamlDb::Dump.should_receive(:dump_table_records)
+ YamlDb::Dump.dump_table(@io, 'mytable')
+ end
+
+ it "should not dump a table's contents when the record count is zero" do
+ YamlDb::Dump.stub!(:table_record_count).with('mytable').and_return(0)
+ YamlDb::Dump.should_not_receive(:dump_table_columns)
+ YamlDb::Dump.should_not_receive(:dump_table_records)
+ YamlDb::Dump.dump_table(@io, 'mytable')
+ end
+end
--- /dev/null
+require File.dirname(__FILE__) + '/base'
+
+describe YamlDb::Load do
+ before do
+ YamlDb::Utils.stub!(:quote_table).with('mytable').and_return('mytable')
+
+ ActiveRecord::Base = mock('ActiveRecord::Base', :null_object => true)
+ ActiveRecord::Base.connection = mock('connection')
+ ActiveRecord::Base.connection.stub!(:transaction).and_yield
+ end
+
+ before(:each) do
+ @io = StringIO.new
+ end
+
+ it "should truncate the table" do
+ ActiveRecord::Base.connection.stub!(:execute).with("TRUNCATE mytable").and_return(true)
+ ActiveRecord::Base.connection.should_not_receive(:execute).with("DELETE FROM mytable")
+ YamlDb::Load.truncate_table('mytable')
+ end
+
+ it "should delete the table if truncate throws an exception" do
+ ActiveRecord::Base.connection.should_receive(:execute).with("TRUNCATE mytable").and_raise()
+ ActiveRecord::Base.connection.should_receive(:execute).with("DELETE FROM mytable").and_return(true)
+ YamlDb::Load.truncate_table('mytable')
+ end
+
+ it "should insert records into a table" do
+ ActiveRecord::Base.connection.stub!(:quote_column_name).with('a').and_return('a')
+ ActiveRecord::Base.connection.stub!(:quote_column_name).with('b').and_return('b')
+ ActiveRecord::Base.connection.stub!(:quote).with(1).and_return("'1'")
+ ActiveRecord::Base.connection.stub!(:quote).with(2).and_return("'2'")
+ ActiveRecord::Base.connection.stub!(:quote).with(3).and_return("'3'")
+ ActiveRecord::Base.connection.stub!(:quote).with(4).and_return("'4'")
+ ActiveRecord::Base.connection.should_receive(:execute).with("INSERT INTO mytable (a,b) VALUES ('1','2')")
+ ActiveRecord::Base.connection.should_receive(:execute).with("INSERT INTO mytable (a,b) VALUES ('3','4')")
+
+ YamlDb::Load.load_records('mytable', ['a', 'b'], [[1, 2], [3, 4]])
+ end
+
+ it "should quote column names that correspond to sql keywords" do
+ ActiveRecord::Base.connection.stub!(:quote_column_name).with('a').and_return('a')
+ ActiveRecord::Base.connection.stub!(:quote_column_name).with('count').and_return('"count"')
+ ActiveRecord::Base.connection.stub!(:quote).with(1).and_return("'1'")
+ ActiveRecord::Base.connection.stub!(:quote).with(2).and_return("'2'")
+ ActiveRecord::Base.connection.stub!(:quote).with(3).and_return("'3'")
+ ActiveRecord::Base.connection.stub!(:quote).with(4).and_return("'4'")
+ ActiveRecord::Base.connection.should_receive(:execute).with("INSERT INTO mytable (a,\"count\") VALUES ('1','2')")
+ ActiveRecord::Base.connection.should_receive(:execute).with("INSERT INTO mytable (a,\"count\") VALUES ('3','4')")
+
+ YamlDb::Load.load_records('mytable', ['a', 'count'], [[1, 2], [3, 4]])
+ end
+
+ it "should truncate the table and then load the records into the table" do
+ YamlDb::Load.should_receive(:truncate_table).with('mytable')
+ YamlDb::Load.should_receive(:load_records).with('mytable', ['a', 'b'], [[1, 2], [3, 4]])
+ YamlDb::Load.should_receive(:reset_pk_sequence!).with('mytable')
+
+ YamlDb::Load.load_table('mytable', { 'columns' => [ 'a', 'b' ], 'records' => [[1, 2], [3, 4]] })
+ end
+
+ it "should call load structure for each document in the file" do
+ YAML.should_receive(:load_documents).with(@io).and_yield({ 'mytable' => {
+ 'columns' => [ 'a', 'b' ],
+ 'records' => [[1, 2], [3, 4]]
+ } })
+ YamlDb::Load.should_receive(:load_table).with('mytable', { 'columns' => [ 'a', 'b' ], 'records' => [[1, 2], [3, 4]] })
+ YamlDb::Load.load(@io)
+ end
+
+ it "should not call load structure when the document in the file contains no records" do
+ YAML.should_receive(:load_documents).with(@io).and_yield({ 'mytable' => nil })
+ YamlDb::Load.should_not_receive(:load_table)
+ YamlDb::Load.load(@io)
+ end
+
+ it "should call reset pk sequence if the connection adapter is postgres" do
+ ActiveRecord::Base.connection.should_receive(:respond_to?).with(:reset_pk_sequence!).and_return(true)
+ ActiveRecord::Base.connection.should_receive(:reset_pk_sequence!).with('mytable')
+ YamlDb::Load.reset_pk_sequence!('mytable')
+ end
+
+ it "should not call reset pk sequence for other adapters" do
+ ActiveRecord::Base.connection.should_receive(:respond_to?).with(:reset_pk_sequence!).and_return(false)
+ ActiveRecord::Base.connection.should_not_receive(:reset_pk_sequence!)
+ YamlDb::Load.reset_pk_sequence!('mytable')
+ end
+end
--- /dev/null
+require File.dirname(__FILE__) + '/base'
+
+describe YamlDb::Utils, " convert records utility method" do
+ before do
+ ActiveRecord::Base = mock('ActiveRecord::Base', :null_object => true)
+ ActiveRecord::Base.connection = mock('connection')
+ end
+
+ it "turns an array with one record into a yaml chunk" do
+ YamlDb::Utils.chunk_records([ %w(a b) ]).should == <<EOYAML
+ - - a
+ - b
+EOYAML
+ end
+
+ it "turns an array with two records into a yaml chunk" do
+ YamlDb::Utils.chunk_records([ %w(a b), %w(x y) ]).should == <<EOYAML
+ - - a
+ - b
+ - - x
+ - y
+EOYAML
+ end
+
+ it "returns an array of hash values using an array of ordered keys" do
+ YamlDb::Utils.unhash({ 'a' => 1, 'b' => 2 }, [ 'b', 'a' ]).should == [ 2, 1 ]
+ end
+
+ it "should unhash each hash an array using an array of ordered keys" do
+ YamlDb::Utils.unhash_records([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ], [ 'b', 'a' ]).should == [ [ 2, 1 ], [ 4, 3 ] ]
+ end
+
+ it "should return true if it is a boolean type" do
+ YamlDb::Utils.is_boolean(true).should == true
+ YamlDb::Utils.is_boolean('true').should_not == true
+ end
+
+ it "should return an array of boolean columns" do
+ ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('a',:name => 'a',:type => :string), mock('b', :name => 'b',:type => :boolean) ])
+ YamlDb::Utils.boolean_columns('mytable').should == ['b']
+ end
+
+ it "should quote the table name" do
+ ActiveRecord::Base.connection.should_receive(:quote_table_name).with('values').and_return('`values`')
+ YamlDb::Utils.quote_table('values').should == '`values`'
+ end
+end
--- /dev/null
+namespace :db do
+ desc "Dump schema and data to db/schema.rb and db/data.yml"
+ task(:dump => [ "db:schema:dump", "db:data:dump" ])
+
+ desc "Load schema and data from db/schema.rb and db/data.yml"
+ task(:load => [ "db:schema:load", "db:data:load" ])
+
+ namespace :data do
+ def db_dump_data_file
+ "#{RAILS_ROOT}/db/data.yml"
+ end
+
+ desc "Dump contents of database to db/data.yml"
+ task(:dump => :environment) do
+ YamlDb.dump db_dump_data_file
+ end
+
+ desc "Load contents of db/data.yml into database"
+ task(:load => :environment) do
+ YamlDb.load db_dump_data_file
+ end
+ end
+end
projects / selectricity / commitdiff