Post Views : 0

Issue

This Content is from Stack Overflow. Question asked by Akshay Kulkarni

I have below type of events. I’m trying to split field’s key, value as new event.
I’m able to do it for two fields(TOTAl_VOLUME, SUCCESS_VOLUME), but when I try for 3rd field, logstash stop responding.

{
           "agentId" => "Log_Agent",
         "@metadata" => {
                  "txnId1" => "GET#/txn/branchserver17.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12",
        "A1EvtFingerprint" => "AGENTID=Log_Agent&TIME=1657708200000&RESPTYPE=DC",
               "indexname" => "heal_collated_agent_txn",
                  "txnId2" => "GET#/txn/branchserver17.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12",
               "tablename" => "agent_transactions_data",
               "accountid" => "mle_account",
        "enable_rubydebug" => "true"
    },
      "max_response" => {
        "GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.011000156402588,
        "GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.011000156402588,
        "GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.011000156402588
    },
     "response_type" => "DC",
    "aggLevelInMins" => 15,
         "timeInGMT" => 1657708200000,
      "avg_response" => {
        "GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 4.5954742431640625,
        "GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 4.6110687255859375,
        "GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 4.580192565917969
    },
           "timeout" => {
        "GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 777,
        "GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 839,
        "GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 781
    },
           "unknown" => {
        "GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 773,
        "GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 794,
        "GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 746
    },
              "fail" => {
        "GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 770,
        "GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 737,
        "GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 800
    },
        "@timestamp" => 2022-07-13T10:30:00.000Z,
              "slow" => {
        "GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 782,
        "GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 788,
        "GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 744
    },
      "min_response" => {
        "GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.0,
        "GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.0,
        "GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.0
    },
            "dcKpis" => {
          "TOTAl_VOLUME" => 10957,
        "SUCCESS_VOLUME" => 7776
    },
          "@version" => "1"
}

Desired output should be(Splitted multiple field value into multiple events)

          "txnId" : "POST#http:/|acc=heal_health",
          "timeInGMT" : 1657048320000,
          "dcKpis" : {
            "UNKNOWN_VOLUME" : 59.0,
            "TIMEOUT_VOLUME" : 59.0,
            "FAIL_VOLUME" : 59.0,
            "MIN_RESPONSE_TIME" : 1000000.0,
            "TOTAL_VOLUME" : 236.0,
            "RESPONSE_TIME" : 1000000.0,
            "SUCCESS_VOLUME" : 59.0,
            "MAX_RESPONSE_TIME" : 1000000.0,
            "SLOW_VOLUME" : 0.0
          },`Preformatted text`

Following is my pipeline:

      ruby {
        code => '
          values =  event.get("total")
          if values.is_a? Hash
            someField1 = []
            values.each { |k, v|
                someField1 << { "txnId1" => k, "total" => v }
            }
            event.set("someField1", someField1)
          end
          event.remove("total")
         '
      }


      ruby {
        code => '
          values =  event.get("success")
          if values.is_a? Hash
            someField2 = []
            values.each { |k, v|
                someField2 << { "txnId2" => k, "success" => v }
            }
            event.set("someField2", someField2)
          end
          event.remove("success")
         '
      }


      split {
        field => 'someField1'
      }
      split {
        field => 'someField2'
      }



      mutate {
        rename => {
          "[someField1][txnId1]" => "[@metadata][txnId1]"
          "[someField1][total]" => "[dcKpis][TOTAl_VOLUME]"
          "[someField2][txnId2]" => "[@metadata][txnId2]"
          "[someField2][success]" => "[dcKpis][SUCCESS_VOLUME]"
        }
        remove_field => ["someField1","someField2","someField3","someField4","someField5","someField6","someField7","someField8","someField9"]
      }


Please suggest if anybody aware about this.

Regards,
Akshay Kulkarnni

Solution

This question is not yet answered, be the first one who answer using the comment. Later the confirmed answer will be published as the solution.

This Question and Answer are collected from stackoverflow and tested by JTuto community, is licensed under the terms of CC BY-SA 2.5. - CC BY-SA 3.0. - CC BY-SA 4.0.