Kimurai

# github_spider.rb
require 'kimurai'

class GithubSpider < Kimurai::Base
  @name = "github_spider"
  @engine = :selenium_chrome
  @start_urls = ["https://github.com/search?q=Ruby%20Web%20Scraping"]
  @config = {
    user_agent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36",
    before_request: { delay: 4..7 }
  }

  def parse(response, url:, data: {})
    response.xpath("//ul[@class='repo-list']/div//h3/a").each do |a|
      request_to :parse_repo_page, url: absolute_url(a[:href], base: url)
    end

    if next_page = response.at_xpath("//a[@class='next_page']")
      request_to :parse, url: absolute_url(next_page[:href], base: url)
    end
  end

  def parse_repo_page(response, url:, data: {})
    item = {}

    item[:owner] = response.xpath("//h1//a[@rel='author']").text
    item[:repo_name] = response.xpath("//h1/strong[@itemprop='name']/a").text
    item[:repo_url] = url
    item[:description] = response.xpath("//span[@itemprop='about']").text.squish
    item[:tags] = response.xpath("//div[@id='topics-list-container']/div/a").map { |a| a.text.squish }
    item[:watch_count] = response.xpath("//ul[@class='pagehead-actions']/li[contains(., 'Watch')]/a[2]").text.squish
    item[:star_count] = response.xpath("//ul[@class='pagehead-actions']/li[contains(., 'Star')]/a[2]").text.squish
    item[:fork_count] = response.xpath("//ul[@class='pagehead-actions']/li[contains(., 'Fork')]/a[2]").text.squish
    item[:last_commit] = response.xpath("//span[@itemprop='dateModified']/*").text

    save_to "results.json", item, format: :pretty_json
  end
end

GithubSpider.crawl!

# infinite_scroll_spider.rb
require 'kimurai'

class InfiniteScrollSpider < Kimurai::Base
  @name = "infinite_scroll_spider"
  @engine = :selenium_chrome
  @start_urls = ["https://infinite-scroll.com/demo/full-page/"]

  def parse(response, url:, data: {})
    posts_headers_path = "//article/h2"
    count = response.xpath(posts_headers_path).count

    loop do
      browser.execute_script("window.scrollBy(0,10000)") ; sleep 2
      response = browser.current_response

      new_count = response.xpath(posts_headers_path).count
      if count == new_count
        logger.info "> Pagination is done" and break
      else
        count = new_count
        logger.info "> Continue scrolling, current count is #{count}..."
      end
    end

    posts_headers = response.xpath(posts_headers_path).map(&:text)
    logger.info "> All posts from page: #{posts_headers.join('; ')}"
  end
end

InfiniteScrollSpider.crawl!

require 'kimurai'

class SimpleSpider < Kimurai::Base
  @name = "simple_spider"
  @engine = :selenium_chrome
  @start_urls = ["https://example.com/"]

  def parse(response, url:, data: {})
  end
end

SimpleSpider.crawl!

def parse(response, url:, data: {})
end

class ProductsSpider < Kimurai::Base
  @engine = :selenium_chrome
  @start_urls = ["https://example-shop.com/example-product-category"]

  def parse(response, url:, data: {})
    category_name = response.xpath("//path/to/category/name").text
    response.xpath("//path/to/products/urls").each do |product_url|
      # Merge category_name with current data hash and pass it next to parse_product method
      request_to(:parse_product, url: product_url[:href], data: data.merge(category_name: category_name))
    end

    # ...
  end

  def parse_product(response, url:, data: {})
    item = {}
    # Assign item's category_name from data[:category_name]
    item[:category_name] = data[:category_name]

    # ...
  end
end

class GoogleSpider < Kimurai::Base
  @name = "google_spider"
  @engine = :selenium_chrome
  @start_urls = ["https://www.google.com/"]

  def parse(response, url:, data: {})
    browser.fill_in "q", with: "Kimurai web scraping framework"
    browser.click_button "Google Search"

    # Update response to current response after interaction with a browser
    response = browser.current_response

    # Collect results
    results = response.xpath("//div[@class='g']//h3/a").map do |a|
      { title: a.text, url: a[:href] }
    end

    # ...
  end
end

class Spider < Kimurai::Base
  @engine = :selenium_chrome
  @start_urls = ["https://example.com/"]

  def parse(response, url:, data: {})
    # Process request to `parse_product` method with `https://example.com/some_product` url:
    request_to :parse_product, url: "https://example.com/some_product"
  end

  def parse_product(response, url:, data: {})
    puts "From page https://example.com/some_product !"
  end
end

def request_to(handler, url:, data: {})
  request_data = { url: url, data: data }

  browser.visit(url)
  public_send(handler, browser.current_response, request_data)
end

class Spider < Kimurai::Base
  @engine = :selenium_chrome
  @start_urls = ["https://example.com/"]

  def parse(response, url:, data: {})
    url_to_process = "https://example.com/some_product"

    browser.visit(url_to_process)
    parse_product(browser.current_response, url: url_to_process)
  end

  def parse_product(response, url:, data: {})
    puts "From page https://example.com/some_product !"
  end
end

class ProductsSpider < Kimurai::Base
  @engine = :selenium_chrome
  @start_urls = ["https://example-shop.com/"]

  # ...

  def parse_product(response, url:, data: {})
    item = {}

    item[:title] = response.xpath("//title/path").text
    item[:description] = response.xpath("//desc/path").text.squish
    item[:price] = response.xpath("//price/path").text[/\d+/]&.to_f

    # Add each new item to the `scraped_products.json` file:
    save_to "scraped_products.json", item, format: :json
  end
end

class ProductsSpider < Kimurai::Base
  @engine = :selenium_chrome
  @start_urls = ["https://example-shop.com/"]

  def parse(response, url:, data: {})
    response.xpath("//categories/path").each do |category|
      request_to :parse_category, url: category[:href]
    end
  end

  # Check products for uniqueness using product url inside of parse_category:
  def parse_category(response, url:, data: {})
    response.xpath("//products/path").each do |product|
      # Skip url if it's not unique:
      next unless unique?(:product_url, product[:href])
      # Otherwise process it:
      request_to :parse_product, url: product[:href]
    end
  end

  # Or/and check products for uniqueness using product sku inside of parse_product:
  def parse_product(response, url:, data: {})
    item = {}
    item[:sku] = response.xpath("//product/sku/path").text.strip.upcase
    # Don't save product and return from method if there is already saved item with the same sku:
    return unless unique?(:sku, item[:sku])

    # ...
    save_to "results.json", item, format: :json
  end
end

# Check string "http://example.com" in scope `url` for a first time:
unique?(:url, "http://example.com")
# => true

# Try again:
unique?(:url, "http://example.com")
# => false

# `product_url` scope
unique?(:product_url, "http://example.com/product_1")

# `id` scope
unique?(:id, 324234232)

# `custom` scope
unique?(:custom, "Lorem Ipsum")

@config = {
  retry_request_errors: [{ error: RuntimeError, skip_on_failure: true }]
}

def parse_product(response, url:, data: {})
  unless response.at_xpath("//path/to/add_to_card_button")
    add_event("Product is sold") and return
  end

  # ...
end

require 'kimurai'

class ExampleSpider < Kimurai::Base
  @name = "example_spider"
  @engine = :selenium_chrome
  @start_urls = ["https://example.com/"]

  def self.open_spider
    logger.info "> Starting..."
  end

  def self.close_spider
    logger.info "> Stopped!"
  end

  def parse(response, url:, data: {})
    logger.info "> Scraping..."
  end
end

ExampleSpider.crawl!

    11: def self.open_spider
 => 12:   binding.pry
    13: end

[1] pry(example_spider)> run_info
=> {
  :spider_name=>"example_spider",
  :status=>:running,
  :environment=>"development",
  :start_time=>2018-08-05 23:32:00 +0400,
  :stop_time=>nil,
  :running_time=>nil,
  :visits=>{:requests=>0, :responses=>0},
  :error=>nil
}

    15: def self.close_spider
 => 16:   binding.pry
    17: end

[1] pry(example_spider)> run_info
=> {
  :spider_name=>"example_spider",
  :status=>:completed,
  :environment=>"development",
  :start_time=>2018-08-05 23:32:00 +0400,
  :stop_time=>2018-08-05 23:32:06 +0400,
  :running_time=>6.214,
  :visits=>{:requests=>1, :responses=>1},
  :error=>nil
}

class ExampleSpider < Kimurai::Base
  @name = "example_spider"
  @engine = :selenium_chrome
  @start_urls = ["https://example.com/"]

  def self.close_spider
    puts ">>> run info: #{run_info}"
  end

  def parse(response, url:, data: {})
    logger.info "> Scraping..."
    # Let's try to strip nil:
    nil.strip
  end
end

class Spider < Kimurai::Base
  @engine = :selenium_chrome
  @start_urls = ["https://example.com/"]

  def self.close_spider
    if completed?
      send_file_to_ftp("results.json")
    else
      send_error_notification(run_info[:error])
    end
  end

  def self.send_file_to_ftp(file_path)
    # ...
  end

  def self.send_error_notification(error)
    # ...
  end

  # ...

  def parse_item(response, url:, data: {})
    item = {}
    # ...

    save_to "results.json", item, format: :json
  end
end

class Spider < Kimurai::Base
  @engine = :selenium_chrome
  @start_urls = ["https://example.com/"]

  def self.close_spider
    if failed? && Kimurai.env == "production"
      send_error_notification(run_info[:error])
    else
      # Do nothing
    end
  end

  # ...
end

# amazon_spider.rb
require 'kimurai'

class AmazonSpider < Kimurai::Base
  @name = "amazon_spider"
  @engine = :mechanize
  @start_urls = ["https://www.amazon.com/"]

  def parse(response, url:, data: {})
    browser.fill_in "field-keywords", with: "Web Scraping Books"
    browser.click_on "Go"

    # Walk through pagination and collect products urls:
    urls = []
    loop do
      response = browser.current_response
      response.xpath("//li//a[contains(@class, 's-access-detail-page')]").each do |a|
        urls << a[:href].sub(/ref=.+/, "")
      end

      browser.find(:xpath, "//a[@id='pagnNextLink']", wait: 1).click rescue break
    end

    # Process all collected urls concurrently within 3 threads:
    in_parallel(:parse_book_page, urls, threads: 3)
  end

  def parse_book_page(response, url:, data: {})
    item = {}

    item[:title] = response.xpath("//h1/span[@id]").text.squish
    item[:url] = url
    item[:price] = response.xpath("(//span[contains(@class, 'a-color-price')])[1]").text.squish.presence
    item[:publisher] = response.xpath("//h2[text()='Product details']/following::b[text()='Publisher:']/following-sibling::text()[1]").text.squish.presence

    save_to "books.json", item, format: :pretty_json
  end
end

AmazonSpider.crawl!

### Settings ###
require 'tzinfo'

# Export current PATH to the cron
env :PATH, ENV["PATH"]

# Use 24 hour format when using `at:` option
set :chronic_options, hours24: true

# Use local_to_utc helper to setup execution time using your local timezone instead
# of server's timezone (which is probably and should be UTC, to check run `$ timedatectl`).
# Also maybe you'll want to set same timezone in kimurai as well (use `Kimurai.configuration.time_zone =` for that),
# to have spiders logs in a specific time zone format.
# Example usage of helper:
# every 1.day, at: local_to_utc("7:00", zone: "Europe/Moscow") do
#   crawl "google_spider.com", output: "log/google_spider.com.log"
# end
def local_to_utc(time_string, zone:)
  TZInfo::Timezone.get(zone).local_to_utc(Time.parse(time_string))
end

# Note: by default Whenever exports cron commands with :environment == "production".
# Note: Whenever can only append log data to a log file (>>). If you want
# to overwrite (>) log file before each run, pass lambda:
# crawl "google_spider.com", output: -> { "> log/google_spider.com.log 2>&1" }

# Project job types
job_type :crawl,  "cd :path && KIMURAI_ENV=:environment bundle exec kimurai crawl :task :output"
job_type :runner, "cd :path && KIMURAI_ENV=:environment bundle exec kimurai runner --jobs :task :output"

# Single file job type
job_type :single, "cd :path && KIMURAI_ENV=:environment ruby :task :output"
# Single with bundle exec
job_type :single_bundle, "cd :path && KIMURAI_ENV=:environment bundle exec ruby :task :output"

### Schedule ###
# Usage (check examples here https://github.com/javan/whenever#example-schedulerb-file):
# every 1.day do
  # Example to schedule a single spider in the project:
  # crawl "google_spider.com", output: "log/google_spider.com.log"

  # Example to schedule all spiders in the project using runner. Each spider will write
  # it's own output to the `log/spider_name.log` file (handled by a runner itself).
  # Runner output will be written to log/runner.log file.
  # Argument number it's a count of concurrent jobs:
  # runner 3, output:"log/runner.log"

  # Example to schedule single spider (without project):
  # single "single_spider.rb", output: "single_spider.log"
# end

### How to set a cron schedule ###
# Run: `$ whenever --update-crontab --load-file config/schedule.rb`.
# If you don't have whenever command, install the gem: `$ gem install whenever`.

### How to cancel a schedule ###
# Run: `$ whenever --clear-crontab --load-file config/schedule.rb`.

every 1.day, at: "7:00" do
  single "example_spider.rb", output: "example_spider.log"
end

Kimurai.configure do |config|
  # Default logger has colored mode in development.
  # If you would like to disable it, set `colorize_logger` to false.
  # config.colorize_logger = false

  # Logger level for default logger:
  # config.log_level = :info

  # Custom logger:
  # config.logger = Logger.new(STDOUT)

  # Custom time zone (for logs):
  # config.time_zone = "UTC"
  # config.time_zone = "Europe/Moscow"

  # Provide custom chrome binary path (default is any available chrome/chromium in the PATH):
  # config.selenium_chrome_path = "/usr/bin/chromium-browser"
  # Provide custom selenium chromedriver path (default is "/usr/local/bin/chromedriver"):
  # config.chromedriver_path = "~/.local/bin/chromedriver"
end

class ExampleSpider < Kimurai::Base
  @name = "example_spider"
  @engine = :mechanize
  @start_urls = ["https://example.com/"]

  def parse(response, url:, data: {})
    title = response.xpath("//title").text.squish
  end
end

ExampleSpider.crawl!
# => { :spider_name => "example_spider", :status => :completed, :environment => "development", :start_time => 2018-08-22 18:20:16 +0400, :stop_time => 2018-08-22 18:20:17 +0400, :running_time => 1.216, :visits => { :requests => 1, :responses => 1 }, :items => { :sent => 0, :processed => 0 }, :error => nil }

2.times do |i|
  Thread.new { p i, ExampleSpider.crawl! }
end # =>

# 1
# false

# 0
# {:spider_name=>"example_spider", :status=>:completed, :environment=>"development", :start_time=>2018-08-22 18:49:22 +0400, :stop_time=>2018-08-22 18:49:23 +0400, :running_time=>0.801, :visits=>{:requests=>1, :responses=>1}, :items=>{:sent=>0, :processed=>0}, :error=>nil}

class ExampleSpider < Kimurai::Base
  @name = "example_spider"
  @engine = :mechanize
  @start_urls = ["https://example.com/"]

  def parse(response, url:, data: {})
    title = response.xpath("//title").text.squish
  end
end

ExampleSpider.parse!(:parse, url: "https://example.com/")
# => "Example Domain"

urls = ["https://www.google.com/", "https://www.reddit.com/", "https://en.wikipedia.org/"]

urls.each do |url|
  Thread.new { p ExampleSpider.parse!(:parse, url: url) }
end # =>

# "Google"
# "Wikipedia, the free encyclopedia"
# "reddit: the front page of the internetHotHot"

class GoogleSpider < Kimurai::Base
  @name = "google_spider"
end

class RedditSpider < Kimurai::Base
  @name = "reddit_spider"
end

class WikipediaSpider < Kimurai::Base
  @name = "wikipedia_spider"
end

# To get the list of all available spider classes:
Kimurai.list
# => {"google_spider"=>GoogleSpider, "reddit_spider"=>RedditSpider, "wikipedia_spider"=>WikipediaSpider}

# To find a particular spider class by it's name:
Kimurai.find_by_name("reddit_spider")
# => RedditSpider

class Spider < Kimurai::Base
  USER_AGENTS = ["Chrome", "Firefox", "Safari", "Opera"]
  PROXIES = ["2.3.4.5:8080:http:username:password", "3.4.5.6:3128:http", "1.2.3.4:3000:socks5"]

  @engine = :poltergeist_phantomjs
  @start_urls = ["https://example.com/"]
  @config = {
    headers: { "custom_header" => "custom_value" },
    cookies: [{ name: "cookie_name", value: "cookie_value", domain: ".example.com" }],
    user_agent: -> { USER_AGENTS.sample },
    proxy: -> { PROXIES.sample },
    window_size: [1366, 768],
    disable_images: true,
    restart_if: {
      # Restart browser if provided memory limit (in kilobytes) is exceeded:
      memory_limit: 350_000
    },
    before_request: {
      # Change user agent before each request:
      change_user_agent: true,
      # Change proxy before each request:
      change_proxy: true,
      # Clear all cookies and set default cookies (if provided) before each request:
      clear_and_set_cookies: true,
      # Process delay before each request:
      delay: 1..3
    }
  }

  def parse(response, url:, data: {})
    # ...
  end
end

@config = {
  # Custom headers, format: hash. Example: { "some header" => "some value", "another header" => "another value" }
  # Works only for :mechanize and :poltergeist_phantomjs engines (Selenium doesn't allow to set/get headers)
  headers: {},

  # Custom User Agent, format: string or lambda.
  # Use lambda if you want to rotate user agents before each run:
  # user_agent: -> { ARRAY_OF_USER_AGENTS.sample }
  # Works for all engines
  user_agent: "Mozilla/5.0 Firefox/61.0",

  # Custom cookies, format: array of hashes.
  # Format for a single cookie: { name: "cookie name", value: "cookie value", domain: ".example.com" }
  # Works for all engines
  cookies: [],

  # Proxy, format: string or lambda. Format of a proxy string: "ip:port:protocol:user:password"
  # `protocol` can be http or socks5. User and password are optional.
  # Use lambda if you want to rotate proxies before each run:
  # proxy: -> { ARRAY_OF_PROXIES.sample }
  # Works for all engines, but keep in mind that Selenium drivers doesn't support proxies
  # with authorization. Also, Mechanize doesn't support socks5 proxy format (only http)
  proxy: "3.4.5.6:3128:http:user:pass",

  # If enabled, browser will ignore any https errors. It's handy while using a proxy
  # with self-signed SSL cert (for example Crawlera or Mitmproxy)
  # Also, it will allow to visit webpages with expires SSL certificate.
  # Works for all engines
  ignore_ssl_errors: true,

  # Custom window size, works for all engines
  window_size: [1366, 768],

  # Skip images downloading if true, works for all engines
  disable_images: true,

  # Selenium engines only: headless mode, `:native` or `:virtual_display` (default is :native)
  # Although native mode has a better performance, virtual display mode
  # sometimes can be useful. For example, some websites can detect (and block)
  # headless chrome, so you can use virtual_display mode instead
  headless_mode: :native,

  # This option tells the browser not to use a proxy for the provided list of domains or IP addresses.
  # Format: array of strings. Works only for :selenium_firefox and selenium_chrome
  proxy_bypass_list: [],

  # Option to provide custom SSL certificate. Works only for :poltergeist_phantomjs and :mechanize
  ssl_cert_path: "path/to/ssl_cert",

  # Inject some JavaScript code to the browser.
  # Format: array of strings, where each string is a path to JS file.
  # Works only for poltergeist_phantomjs engine (Selenium doesn't support JS code injection)
  extensions: ["lib/code_to_inject.js"],

  # Automatically skip duplicated (already visited) urls when using `request_to` method.
  # Possible values: `true` or `hash` with options.
  # In case of `true`, all visited urls will be added to the storage's scope `:requests_urls`
  # and if url already contains in this scope, request will be skipped.
  # You can configure this setting by providing additional options as hash:
  # `skip_duplicate_requests: { scope: :custom_scope, check_only: true }`, where:
  # `scope:` - use custom scope than `:requests_urls`
  # `check_only:` - if true, then scope will be only checked for url, url will not
  # be added to the scope if scope doesn't contains it.
  # works for all drivers
  skip_duplicate_requests: true,

  # Automatically skip provided errors while requesting a page.
  # If raised error matches one of the errors in the list, then this error will be caught,
  # and request will be skipped.
  # It is a good idea to skip errors like NotFound(404), etc.
  # Format: array where elements are error classes or/and hashes. You can use hash format
  # for more flexibility: `{ error: "RuntimeError", message: "404 => Net::HTTPNotFound" }`.
  # Provided `message:` will be compared with a full error message using `String#include?`. Also
  # you can use regex instead: `{ error: "RuntimeError", message: /404|403/ }`.
  skip_request_errors: [{ error: RuntimeError, message: "404 => Net::HTTPNotFound" }],

  # Automatically retry provided errors with a few attempts while requesting a page.
  # If raised error matches one of the errors in the list, then this error will be caught
  # and the request will be processed again within a delay. There are 3 attempts:
  # first: delay 15 sec, second: delay 30 sec, third: delay 45 sec.
  # If after 3 attempts there is still an exception, then the exception will be raised.
  # It is a good idea to try to retry errros like `ReadTimeout`, `HTTPBadGateway`, etc.
  # Format: same like for `skip_request_errors` option.
  retry_request_errors: [Net::ReadTimeout],

  # Handle page encoding while parsing html response using Nokogiri. There are two modes:
  # Auto (`:auto`) (try to fetch correct encoding from <meta http-equiv="Content-Type"> or <meta charset> tags)
  # Set required encoding manually, example: `encoding: "GB2312"` (Set required encoding manually)
  # Default this option is unset.
  encoding: nil,

  # Restart browser if one of the options is true:
  restart_if: {
    # Restart browser if provided memory limit (in kilobytes) is exceeded (works for all engines)
    memory_limit: 350_000,

    # Restart browser if provided requests limit is exceeded (works for all engines)
    requests_limit: 100
  },

  # Perform several actions before each request:
  before_request: {
    # Change proxy before each request. The `proxy:` option above should be presented
    # and has lambda format. Works only for poltergeist and mechanize engines
    # (Selenium doesn't support proxy rotation).
    change_proxy: true,

    # Change user agent before each request. The `user_agent:` option above should be presented
    # and has lambda format. Works only for poltergeist and mechanize engines
    # (selenium doesn't support to get/set headers).
    change_user_agent: true,

    # Clear all cookies before each request, works for all engines
    clear_cookies: true,

    # If you want to clear all cookies + set custom cookies (`cookies:` option above should be presented)
    # use this option instead (works for all engines)
    clear_and_set_cookies: true,

    # Global option to set delay between requests.
    # Delay can be `Integer`, `Float` or `Range` (`2..5`). In case of a range,
    # delay number will be chosen randomly for each request: `rand (2..5) # => 3`
    delay: 1..3
  }
}

class ApplicationSpider < Kimurai::Base
  @engine = :poltergeist_phantomjs
  @config = {
    user_agent: "Firefox",
    disable_images: true,
    restart_if: { memory_limit: 350_000 },
    before_request: { delay: 1..2 }
  }
end

class CustomSpider < ApplicationSpider
  @name = "custom_spider"
  @start_urls = ["https://example.com/"]
  @config = {
    before_request: { delay: 4..6 }
  }

  def parse(response, url:, data: {})
    # ...
  end
end

class ExampleSpider < ApplicationSpider
  @name = "example_spider"
  @start_urls = []
  @config = {}

  def parse(response, url:, data: {})
  end
end

class Validator < Kimurai::Pipeline
  def process_item(item, options: {})
    # Here you can validate item and raise `DropItemError`
    # if one of the validations failed. Examples:

    # Drop item if it's category is not "shoe":
    if item[:category] != "shoe"
      raise DropItemError, "Wrong item category"
    end

    # Check item sku for uniqueness using buit-in unique? helper:
    unless unique?(:sku, item[:sku])
      raise DropItemError, "Item sku is not unique"
    end

    # Drop item if title length shorter than 5 symbols:
    if item[:title].size < 5
      raise DropItemError, "Item title is short"
    end

    # Drop item if price is not present
    unless item[:price].present?
      raise DropItemError, "item price is not present"
    end

    # Drop item if it doesn't contains any images:
    unless item[:images].present?
      raise DropItemError, "Item images are not present"
    end

    # Pass item to the next pipeline (if it wasn't dropped):
    item
  end
end

class Saver < Kimurai::Pipeline
  def process_item(item, options: {})
    # Here you can save item to the database, send it to a remote API or
    # simply save item to a file format using `save_to` helper:

    # To get the name of current spider: `spider.class.name`
    save_to "db/#{spider.class.name}.json", item, format: :json

    item
  end
end

class ApplicationSpider < Kimurai::Base
  @engine = :selenium_chrome
  # Define pipelines (by order) for all spiders:
  @pipelines = [:validator, :saver]
end

class ShopSpiderOne < ApplicationSpider
  @name = "shop_spider_1"
  @start_urls = ["https://shop-1.com"]

  # ...

  def parse_product(response, url:, data: {})
    # ...

    # Send item to pipelines:
    send_item item
  end
end

class ShopSpiderTwo < ApplicationSpider
  @name = "shop_spider_2"
  @start_urls = ["https://shop-2.com"]

  def parse_product(response, url:, data: {})
    # ...

    # Send item to pipelines:
    send_item item
  end
end

class ShopSpiderThree < ApplicationSpider
  @name = "shop_spider_3"
  @start_urls = ["https://shop-3.com"]

  def parse_product(response, url:, data: {})
    # ...

    # Send item to pipelines:
    send_item item
  end
end

class Validator < Kimurai::Pipeline
  def process_item(item, options: {})
    if item[:star_count] < 10
      raise DropItemError, "Repository doesn't have enough stars"
    end

    item
  end
end

class GithubSpider < ApplicationSpider
  @name = "github_spider"
  @engine = :selenium_chrome
  @pipelines = [:validator]
  @start_urls = ["https://github.com/search?q=Ruby%20Web%20Scraping"]
  @config = {
    user_agent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36",
    before_request: { delay: 4..7 }
  }

  def parse(response, url:, data: {})
    response.xpath("//ul[@class='repo-list']/div//h3/a").each do |a|
      request_to :parse_repo_page, url: absolute_url(a[:href], base: url)
    end

    if next_page = response.at_xpath("//a[@class='next_page']")
      request_to :parse, url: absolute_url(next_page[:href], base: url)
    end
  end

  def parse_repo_page(response, url:, data: {})
    item = {}

    item[:owner] = response.xpath("//h1//a[@rel='author']").text
    item[:repo_name] = response.xpath("//h1/strong[@itemprop='name']/a").text
    item[:repo_url] = url
    item[:description] = response.xpath("//span[@itemprop='about']").text.squish
    item[:tags] = response.xpath("//div[@id='topics-list-container']/div/a").map { |a| a.text.squish }
    item[:watch_count] = response.xpath("//ul[@class='pagehead-actions']/li[contains(., 'Watch')]/a[2]").text.squish.delete(",").to_i
    item[:star_count] = response.xpath("//ul[@class='pagehead-actions']/li[contains(., 'Star')]/a[2]").text.squish.delete(",").to_i
    item[:fork_count] = response.xpath("//ul[@class='pagehead-actions']/li[contains(., 'Fork')]/a[2]").text.squish.delete(",").to_i
    item[:last_commit] = response.xpath("//span[@itemprop='dateModified']/*").text

    send_item item
  end
end

class CustomSpider < ApplicationSpider
  @name = "custom_spider"
  @start_urls = ["https://example.com"]
  @pipelines = [:validator]

  # ...

  def parse_item(response, url:, data: {})
    # ...

    # Pass custom option `skip_uniq_checking` for Validator pipeline:
    send_item item, validator: { skip_uniq_checking: true }
  end
end

class Validator < Kimurai::Pipeline
  def process_item(item, options: {})

    # Do not check item sku for uniqueness if options[:skip_uniq_checking] is true
    if options[:skip_uniq_checking] != true
      raise DropItemError, "Item sku is not unique" unless unique?(:sku, item[:sku])
    end
  end
end