Skip to content

Instantly share code, notes, and snippets.

@rshiva

rshiva/logstash_csv_to_elasticsearch.conf

Created July 2, 2020 19:00
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save rshiva/5acdf1616c1e46061a0c3f98feaad15d to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save rshiva/5acdf1616c1e46061a0c3f98feaad15d to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. rshiva revised this gist Jul 2, 2020. No changes.
  2. rshiva created this gist Jul 2, 2020.
    103 changes: 103 additions & 0 deletions logstash_csv_to_elasticsearch.conf
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,103 @@
    input {
    file {
    path => "file_name.csv"
    start_position => "beginning"
    sincedb_path => "/tmp/mysincedbfile"
    }
    }
    filter {
    csv {
    separator => ","
    columns => ["document_id", "type", "index", "product_id", "image", "alternate_name",
    "name_suggest", "brand_suggest","name_autocomplete","brand_autocomplete",
    "created_at","product_size_id","ean_code","weight","price","is_deleted","deleted_at","product_update_on",
    "product_update_status",
    "in_stock","description","category_id","category_name","category_image","sub_category_id",
    "sub_category_name","sub_category_image","sub_category_is_selected","sub_category_created_at",
    "sub_category_updated_at","brand_id","brand_name","brand_image","brand_is_selected",
    "brand_created_at","brand_updated_at"]
    }
    mutate {
    convert => {"product_id" => "integer"
    "document_id" => "integer"
    "price" => "float"
    "weight" => "float"
    "product_size_id" => "integer"
    "in_stock" => "boolean"
    "is_deleted" => "boolean"
    "product_update_status" => "integer"
    "brand_is_selected" => "boolean"
    "brand_id" => "integer"
    "category_id" => "integer"
    "sub_category_id" => "integer"
    "sub_category_is_selected" => "boolean"
    "ean_code" => "string"
    #"uom" => "keyword"
    }
    }
    mutate{
    gsub => [ "index", "development_shop_inventory_", ""]
    }
    mutate{
    remove_field => ["message","host", "version", "type", "path" , "document_id"]
    }
    date {
    match => ["created_at", "yyyy-MM-dd'T'HH:mm:ss.SSS+05:30"]
    target => "created_at"
    }
    date {
    match => ["deleted_at", "yyyy-MM-dd'T'HH:mm:ss.SSS+05:30"]
    target => "deleted_at"
    }
    date {
    match => ["brand_created_at", "yyyy-MM-dd'T'HH:mm:ss.SSS+05:30"]
    target => "brand_created_at"
    }
    date {
    match => ["sub_category_created_at", "yyyy-MM-dd'T'HH:mm:ss.SSS+05:30"]
    target => "sub_category_created_at"
    }
    date {
    match => ["sub_category_updated_at", "yyyy-MM-dd'T'HH:mm:ss.SSS+05:30"]
    target => "sub_category_updated_at"
    }
    date {
    match => ["brand_updated_at", "yyyy-MM-dd'T'HH:mm:ss.SSS+05:30"]
    target => "brand_updated_at"
    }
    date {
    match => ["product_update_on", "yyyy-MM-dd'T'HH:mm:ss.SSS+05:30"]
    target => "product_update_on"
    }
    mutate {
    rename => {
    "index" => "shop_id"
    "category_id" => "[category][id]"
    "category_name" => "[category][name]"
    "category_image" => "[category][image]"
    "sub_category_id" => "[sub_category][id]"
    "sub_category_name" => "[sub_category][name]"
    "sub_category_image" => "[sub_category][image]"
    "sub_category_is_selected" => "[sub_category][is_selected]"
    "sub_category_created_at" => "[sub_category][created_at]"
    "sub_category_updated_at" => "[sub_category][updated_at]"
    "brand_id" => "[brand][id]"
    "brand_name" => "[brand][name]"
    "brand_image" => "[brand][image]"
    "brand_is_selected" => "[brand][is_selected]"
    "brand_created_at" => "[brand][created_at]"
    "brand_updated_at" => "[brand][updated_at]"
    }
    }
    }
    output {
    elasticsearch {
    hosts => "http://localhost:9200"
    index => "development_shop_inventory101"
    document_id => "%{shop_id}_%{product_size_id}"
    document_type => "new_shop_inventory"
    }
    # stdout {}

    stdout { codec => rubydebug }
    }