TokyoTyrant Ruby Client¶ ↑

This is a c extension for Ruby to access TokyoTyrant databases. It currently supports key/value, table databases and table queries.

# install tokyocabinet (1.4.47) and tokyotyrant (requires 1.1.41)
# after installing tc and tt on linux I had to /sbin/ldconfig (as root)
sudo gem install --no-ri --no-rdoc ruby-tokyotyrant

This is not in production but the initial benchmarks are very interesting. Results look closer to the memcached gem than any other tyrant client I’ve seen for Ruby.

• Key/Value Store: gist.github.com/75212

• Table Store: gist.github.com/74116

• Bulk Operations: gist.github.com/83194

• Bulk Table Operations: gist.github.com/87215

# start tyrant like so:
# ttserver example.tch

require 'tokyo_tyrant'
db = TokyoTyrant::DB.new('127.0.0.1', 1978)

db['foo'] = 'Bar' # => "Bar"
db['foo']         # => "Bar"

db.each{ |k,v| puts [k, v].inspect }
# ["foo", "Bar"]
# => nil

db.mput("1"=>"number_1", "2"=>"number_2", "3"=>"number_3", "4"=>"number_4", "5"=>"number_5")
db.mget(1..3) # => {"1"=>"number_1", "2"=>"number_2", "3"=>"number_3"}

# start tyrant like so:
# ttserver example.tcb

require 'tokyo_tyrant'
bdb = TokyoTyrant::BDB.new('127.0.0.1', 1978)

bdb.putdup('foo', 'bar')
# => true

bdb.putlist({ 'foo' => ['baz', 'bat']})
# => []

bdb.getlist('foo')
# => {"foo"=>["bar", "baz", "bat"]}

bdb.each{ |k,v| puts [k, v].inspect }
# ["foo", "bar"]
# ["foo", "baz"]
# ["foo", "bat"]

# start tyrant like so:
# ttserver example.tct

require 'tokyo_tyrant'
t = TokyoTyrant::Table.new('127.0.0.1', 1978)

t['bar'] = { :baz => 'box' } # => { :baz => 'box' }
t['bar']                     # => { :baz => 'box' }

t.each{ |k,v| puts [k, v].inspect }
# ["bar", {:baz=>"box"}]
# => nil

# bulk operations
h = {}
100.times do |i|
  h[i] = { :name => 'Pat', :sex => i % 2 > 0 ? 'male' : 'female' }
end
t.mput(h)
t.mget(0..3)
# => {"0"=>{:name=>"Pat", :sex=>"female"}, "1"=>{:name=>"Pat", :sex=>"male"}, "2"=>{:name=>"Pat", :sex=>"female"}, "3"=>{:name=>"Pat", :sex=>"male"}}

require 'tokyo_tyrant'
t = TokyoTyrant::Table.new('127.0.0.1', 1978)

100.times do |i|
  t[i] = { 'name' => "Pat #{i}", 'sex' => i % 2 > 0 ? 'male' : 'female' }
end

q = t.query
q.condition('sex', :streq, 'male')
q.limit(5)
# Get a list of IDs
ids = q.search
# => ["1", "3", "5", "7", "9"]
q.order_by(:name, :strdesc)
ids = q.search
# => ["99", "97", "95", "93", "91"]
# Get the actual records
q.get
# => [{:__id=>"99", :sex=>"male", :name=>"Pat 99"}, {:__id=>"97", :sex=>"male", :name=>"Pat 97"}, {:__id=>"95", :sex=>"male", :name=>"Pat 95"}, {:__id=>"93", :sex=>"male", :name=>"Pat 93"}, {:__id=>"91", :sex=>"male", :name=>"Pat 91"}]

# Alternative Syntax (better)

# Query using a block
t.query{ |q|
  q.condition('sex', :streq, 'male')
  q.limit(5)
}
# => ["1", "3", "5", "7", "9"]

# Get records for a query
t.find{ |q|
  q.condition('sex', :streq, 'male')
  q.limit(5)
}
# => [{:sex=>"male", :name=>"Pat 1", :__id=>"1"}, {:sex=>"male", :name=>"Pat 3", :__id=>"3"}, {:sex=>"male", :name=>"Pat 5", :__id=>"5"}, {:sex=>"male", :name=>"Pat 7", :__id=>"7"}, {:sex=>"male", :name=>"Pat 9", :__id=>"9"}]

# Prepare but don't run a search, return a prepared query object
q = t.prepare_query{ |q|
  q.condition('sex', :streq, 'male')
  q.limit(5)
}
# => #<TokyoTyrant::Query:0x247c14 @rdb=#<Object:0x2800a0>, @rdbquery=#<Object:0x247c00>>
q.search
# => ["1", "3", "5", "7", "9"]
q.get
# => [{:sex=>"male", :name=>"Pat 1", :__id=>"1"}, {:sex=>"male", :name=>"Pat 3", :__id=>"3"}, {:sex=>"male", :name=>"Pat 5", :__id=>"5"}, {:sex=>"male", :name=>"Pat 7", :__id=>"7"}, {:sex=>"male", :name=>"Pat 9", :__id=>"9"}]

require 'tokyo_tyrant'
require 'nokogiri'
require 'open-uri'

t = TokyoTyrant::Table.new('127.0.0.1', 1978)

(1..13).each do |n|
  doc = Nokogiri::HTML(open("http://www.sacred-texts.com/chr/herm/hermes#{n}.htm"))
  chapter = doc.css('h2').last.inner_text.gsub(/\n/, '').gsub(/ +/, ' ').strip
  doc.css('p').each_with_index do |paragraph, i|
    paragraph = paragraph.inner_text.gsub(/\n/, '').gsub(/ +/, ' ').strip
    key = "chapter:#{n}:paragraph:#{i+1}"
    t[key] = { :chapter => chapter, :paragraph => paragraph }
  end
end

# full-text search with the phrase of
t.query{ |q| q.condition(:paragraph, :fts, 'rebirth') }
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:5", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:44", "chapter:13:paragraph:57", "chapter:13:paragraph:69", "chapter:13:paragraph:125"]

# full-text search with all tokens in
t.query{ |q| q.condition(:paragraph, :ftsand, 'logos word') }
# => ["chapter:1:paragraph:12", "chapter:1:paragraph:14", "chapter:1:paragraph:17", "chapter:1:paragraph:19", "chapter:1:paragraph:24", "chapter:1:paragraph:27", "chapter:1:paragraph:43", "chapter:1:paragraph:53", "... lots more ..."]

# full-text search with at least one token in
t.query{ |q| q.condition(:paragraph, :ftsor, 'sermon key') }
# => ["chapter:5:paragraph:1", "chapter:9:paragraph:3", "chapter:10:paragraph:1", "chapter:10:paragraph:4", "chapter:10:paragraph:28", "chapter:11:paragraph:3", "chapter:11:paragraph:66", "chapter:11:paragraph:69", "... lots more ..."]

# negated full-text search with at least one token in
t.query{ |q| q.condition(:paragraph, '!ftsor', 'the god he and I that said') }
# => ["chapter:1:paragraph:95", "chapter:1:paragraph:96", "chapter:1:paragraph:97", "chapter:1:paragraph:98", "chapter:1:paragraph:99", "chapter:2:paragraph:3", "chapter:2:paragraph:5", "chapter:2:paragraph:6", "... lots more ..."]

query1 = t.prepare_query{ |q| q.condition(:paragraph, :fts, 'rebirth') }
query2 = t.prepare_query{ |q| q.condition(:paragraph, :fts, 'logos') }

# Get the union of two query sets (OR)
t.search(:union, query1, query2)
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:5", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:44", "chapter:13:paragraph:57", "... lots more ..."]

# Get the intersection of two query sets (AND)
t.search(:intersection, query1, query2)
# => ["chapter:13:paragraph:5", "chapter:13:paragraph:44", "chapter:13:paragraph:69"]

# Get the difference of two query sets (ANDNOT)
t.search(:diff, query1, query2)
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:57", "chapter:13:paragraph:125"]

require 'tokyo_tyrant'
require 'faker'
tyrants = { 1 => TokyoTyrant::Table.new('127.0.0.1', 1978),
            2 => TokyoTyrant::Table.new('127.0.0.1', 1979) }

# dummy data
tyrants.each{ |account_id, table|
  table.clear
  20.times do |i|
    table["#{account_id}/#{i}"] = { # consistent hashing would be good here
      :name => Faker::Company.name,
      :plan => rand(3),
    }
  end
}

queries = tyrants.collect{ |account_id, table|
  table.prepare_query{ |q| q.condition 'plan', :numlt, '1' }
}

TokyoTyrant::Query.parallel_search(*queries).collect{ |r| {r[""] => r["name"]} }
# => [{"1/3"=>"Zemlak-Jerde"}, {"1/6"=>"O'Conner-Batz"}, {"1/9"=>"Kutch, Erdman and Aufderhar"}, {"1/11"=>"Bartoletti, Armstrong and Barrows"}, {"1/12"=>"Ferry-Dicki"}, {"2/0"=>"Schultz-O'Hara"}, {"2/1"=>"Emmerich, Feest and Huels"}, {"2/2"=>"Borer and Sons"}, {"2/3"=>"D'Amore Inc"}, {"2/5"=>"Koch and Sons"}, {"2/8"=>"Schaefer Group"}, {"2/11"=>"Stroman, Toy and Abernathy"}, {"2/19"=>"Gaylord, Reinger and White"}]

# ttserver -ext spec/ext.lua
require 'tokyo_tyrant'
t = TokyoTyrant::Table.new('127.0.0.1', 1978)

t.run(:echo, 'hello', 'world')
# => "hello\tworld"

# usage is similar to single node
require 'tokyo_tyrant/balancer'

servers = ['127.0.0.1:1978',
           '127.0.0.1:1979',
           '127.0.0.1:1980',
           '127.0.0.1:1981']

tb = TokyoTyrant::Balancer::Table.new(servers)

# store server is determined by key which is consistent
tb[:foo] = { 'foo' => 'bar' }
tb[:bar] = { 'bar' => 'baz' }

# retrieval server is determined by key which is consistent
tb[:foo]
# => { 'foo' => 'bar' }

# aggregate from all nodes
tb.size

# parallel_search based querying across all nodes
tb.find{ |q| q.condition(:foo, :streq, 'bar') }

• Flinn Mueller (actsasflinn) author/maintainer

• ??? (gottlike) ruby 1.9.2 compatibility

• Justin Reagor (cheapRoc) specs

• Seth Yates (sethyates) run method (lua ext)

• John Mettraux (jmettraux) inspiration (rufus-tokyo)