From 7711588bf687830d7c3f0c85c73cbc5222f061e1 Mon Sep 17 00:00:00 2001 From: skytelan3 Date: Tue, 13 Aug 2019 17:02:12 +0900 Subject: [PATCH 1/4] =?UTF-8?q?octets,=20packets=20=EA=B3=84=EC=82=B0=20?= =?UTF-8?q?=EB=B0=8F=20=ED=95=84=EB=93=9C=EB=AA=85=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/logstash/codecs/sflow.rb | 35 +++++++----- lib/logstash/codecs/sflow/counter_record.rb | 2 +- lib/logstash/codecs/sflow/datagram.rb | 8 +-- lib/logstash/codecs/sflow/flow_record.rb | 62 ++++++++++----------- lib/logstash/codecs/sflow/packet_header.rb | 38 ++++++------- lib/logstash/codecs/sflow/sample.rb | 28 +++++----- 6 files changed, 89 insertions(+), 84 deletions(-) diff --git a/lib/logstash/codecs/sflow.rb b/lib/logstash/codecs/sflow.rb index 7f2dc0f..c145453 100644 --- a/lib/logstash/codecs/sflow.rb +++ b/lib/logstash/codecs/sflow.rb @@ -54,25 +54,25 @@ def assign_key_value(event, bindata_kv) # @param [Object] sample # @param [Object] record def common_sflow(event, decoded, sample) - event.set('agent_ip', decoded['agent_ip'].to_s) + event.set('deviceIp', decoded['deviceIp'].to_s) assign_key_value(event, decoded) assign_key_value(event, sample) end def snmp_call(event) if @snmp_interface - if event.include?('source_id_type') and event.get('source_id_type').to_s == '0' - if event.include?('source_id_index') - event.set('source_id_index_descr', @snmp.get_interface(event.get('agent_ip'), event.get('source_id_index'))) + if event.include?('sourceIdType') and event.get('sourceIdType').to_s == '0' + if event.include?('sourceIdIndex') + event.set('source_id_index_descr', @snmp.get_interface(event.get('deviceIp'), event.get('sourceIdIndex'))) end - if event.include?('input_interface') - event.set('input_interface_descr', @snmp.get_interface(event.get('agent_ip'), event.get('input_interface'))) + if event.include?('input') + event.set('input_interface_descr', @snmp.get_interface(event.get('deviceIp'), event.get('input'))) end - if event.include?('output_interface') - event.set('output_interface_descr', @snmp.get_interface(event.get('agent_ip'), event.get('output_interface'))) + if event.include?('output') + event.set('output_interface_descr', @snmp.get_interface(event.get('deviceIp'), event.get('output'))) end if event.include?('interface_index') - event.set('interface_index_descr', @snmp.get_interface(event.get('agent_ip'), event.get('interface_index'))) + event.set('interface_index_descr', @snmp.get_interface(event.get('deviceIp'), event.get('interface_index'))) end end end @@ -131,15 +131,20 @@ def decode(payload) assign_key_value(event, record) end - #compute frame_length_times_sampling_rate + #compute octets if event.include?('frame_length') and event.include?('sampling_rate') - event.set('frame_length_times_sampling_rate', event.get('frame_length').to_i * event.get('sampling_rate').to_i) + event.set('octets', event.get('frame_length').to_i * event.get('sampling_rate').to_i) + end + + #compute packets + if event.include?('packets') and event.include?('samplingRate') + event.set('packets', event.get('packets').to_i * event.get('samplingRate').to_i) end if sample['sample_format'] == 1 - event.set('sflow_type', 'flow_sample') + event.set('sflowType', 'flow_sample') else - event.set('sflow_type', 'expanded_flow_sample') + event.set('sflowType', 'expanded_flow_sample') end #Get interface dfescr if snmp_interface true @@ -163,9 +168,9 @@ def decode(payload) assign_key_value(event, record) if sample['sample_format'] == 2 - event.set('sflow_type', 'counter_sample') + event.set('sflowType', 'counter_sample') else - event.set('sflow_type', 'expanded_counter_sample') + event.set('sflowType', 'expanded_counter_sample') end diff --git a/lib/logstash/codecs/sflow/counter_record.rb b/lib/logstash/codecs/sflow/counter_record.rb index 5f943ec..a446b56 100644 --- a/lib/logstash/codecs/sflow/counter_record.rb +++ b/lib/logstash/codecs/sflow/counter_record.rb @@ -89,7 +89,7 @@ class HundredBaseVG < BinData::Record # noinspection RubyResolve class Vlan < BinData::Record endian :big - uint32 :vlan_id + uint32 :vlanId uint64 :octets uint32 :ucastPkts uint32 :multicastPkts diff --git a/lib/logstash/codecs/sflow/datagram.rb b/lib/logstash/codecs/sflow/datagram.rb index 4d46111..25d514d 100644 --- a/lib/logstash/codecs/sflow/datagram.rb +++ b/lib/logstash/codecs/sflow/datagram.rb @@ -14,14 +14,14 @@ class SFlowHeader < BinData::Record class SFlow < BinData::Record endian :big uint32 :sflow_version - uint32 :ip_version - choice :agent_ip, :selection => :ip_version do + uint32 :ipVersion + choice :deviceIp, :selection => :ipVersion do sflow_ip4_addr 1 sflow_ip6_addr 2 end - uint32 :sub_agent_id + uint32 :subAgentId uint32 :sequence_number - uint32 :uptime_in_ms + uint32 :uptimeInMs uint32 :sample_count array :samples, :initial_length => :sample_count do bit20 :sample_entreprise diff --git a/lib/logstash/codecs/sflow/flow_record.rb b/lib/logstash/codecs/sflow/flow_record.rb index f9e8266..7278e5b 100644 --- a/lib/logstash/codecs/sflow/flow_record.rb +++ b/lib/logstash/codecs/sflow/flow_record.rb @@ -10,11 +10,11 @@ class RawPacketHeader < BinData::Buffer default_parameters :length => :record_length endian :big - uint32 :protocol + uint32 :headerProtocol uint32 :frame_length uint32 :stripped uint32 :header_size - choice :sample_header, :selection => :protocol do + choice :sample_header, :selection => :headerProtocol do ethernet_header 1, :size_header => lambda { header_size * 8 } ipv4_header 11, :size_header => lambda { header_size * 8 } ipv6_header 12, :size_header => lambda { header_size * 8 } @@ -26,9 +26,9 @@ class RawPacketHeader < BinData::Buffer class EthernetFrameData < BinData::Record endian :big uint32 :packet_length - sflow_mac_address :src_mac + sflow_mac_address :srcMac skip :length => 2 - sflow_mac_address :dst_mac + sflow_mac_address :dstMac skip :length => 2 uint32 :eth_type end @@ -37,12 +37,12 @@ class EthernetFrameData < BinData::Record class IP4Data < BinData::Record endian :big uint32 :ip_packet_length - uint32 :ip_protocol - sflow_ip4_addr :src_ip - sflow_ip4_addr :dst_ip - uint32 :src_port - uint32 :dst_port - uint32 :tcp_flags + uint32 :protocol + sflow_ip4_addr :srcIpv4 + sflow_ip4_addr :dstIpv4 + uint32 :srcPort + uint32 :dstPort + uint32 :tcpFlags uint32 :ip_type end @@ -51,40 +51,40 @@ class IP6Data < BinData::Record endian :big uint32 :ip_packet_length uint32 :ip_next_header - sflow_ip6_addr :src_ip - sflow_ip6_addr :dst_ip - uint32 :src_port - uint32 :dst_port - uint32 :tcp_flags + sflow_ip6_addr :srcIpv6 + sflow_ip6_addr :dstIpv6 + uint32 :srcPort + uint32 :dstPort + uint32 :tcpFlags uint32 :ip_priority end # noinspection RubyResolve class ExtendedSwitchData < BinData::Record endian :big - uint32 :src_vlan - uint32 :src_priority - uint32 :dst_vlan - uint32 :dst_priority + uint32 :srcVlan + uint32 :srcPriority + uint32 :dstVlan + uint32 :dstPriority end # noinspection RubyResolve class ExtendedRouterData < BinData::Record endian :big - uint32 :ip_version - choice :ip_address_next_hop_router, :selection => :ip_version do + uint32 :ipVersion + choice :ip_address_next_hop_router, :selection => :ipVersion do sflow_ip4_addr 1 sflow_ip6_addr 2 end - uint32 :src_mask_len - uint32 :dst_mask_len + uint32 :srcMask + uint32 :dstMask end # noinspection RubyResolve class ExtendedGatewayData < BinData::Record endian :big - uint32 :ip_version - choice :ip_address_next_hop_router, :selection => :ip_version do + uint32 :ipVersion + choice :ip_address_next_hop_router, :selection => :ipVersion do sflow_ip4_addr 1 sflow_ip6_addr 2 end @@ -122,8 +122,8 @@ class ExtendedUrlData < BinData::Record # noinspection RubyResolve class ExtendedMplsData < BinData::Record endian :big - uint32 :ip_version - choice :ip_address_next_hop_router, :selection => :ip_version do + uint32 :ipVersion + choice :ip_address_next_hop_router, :selection => :ipVersion do sflow_ip4_addr 1 sflow_ip6_addr 2 end @@ -136,13 +136,13 @@ class ExtendedMplsData < BinData::Record # noinspection RubyResolve class ExtendedNatData < BinData::Record endian :big - uint32 :src_ip_version - choice :src_ip_address, :selection => :src_ip_version do + uint32 :srcIpVersion + choice :srcIpAddress, :selection => :srcIpVersion do sflow_ip4_addr 1 sflow_ip6_addr 2 end - uint32 :dst_ip_version - choice :dst_ip_address, :selection => :dst_ip_version do + uint32 :dstIpVersion + choice :dstIpAddress, :selection => :dstIpVersion do sflow_ip4_addr 1 sflow_ip6_addr 2 end diff --git a/lib/logstash/codecs/sflow/packet_header.rb b/lib/logstash/codecs/sflow/packet_header.rb index 769be06..dc6b08d 100644 --- a/lib/logstash/codecs/sflow/packet_header.rb +++ b/lib/logstash/codecs/sflow/packet_header.rb @@ -18,8 +18,8 @@ class TcpHeader < BinData::Record mandatory_parameter :size_header endian :big - uint16 :src_port - uint16 :dst_port + uint16 :srcPort + uint16 :dstPort uint32 :tcp_seq_number uint32 :tcp_ack_number bit4 :tcp_header_length # times 4 @@ -51,8 +51,8 @@ class UdpHeader < BinData::Record mandatory_parameter :size_header endian :big - uint16 :src_port - uint16 :dst_port + uint16 :srcPort + uint16 :dstPort uint16 :udp_length uint16 :udp_checksum bit :data, :nbits => lambda { size_header - 64 } #skip udp data @@ -63,7 +63,7 @@ class IPV4Header < BinData::Record mandatory_parameter :size_header endian :big - bit4 :ip_version + bit4 :ipVersion bit4 :ip_header_length # times 4 bit6 :ip_dscp bit2 :ip_ecn @@ -72,14 +72,14 @@ class IPV4Header < BinData::Record bit3 :ip_flags bit13 :ip_fragment_offset uint8 :ip_ttl - uint8 :ip_protocol + uint8 :protocol uint16 :ip_checksum - sflow_ip4_addr :src_ip - sflow_ip4_addr :dst_ip - array :ip_options, :initial_length => lambda { (((ip_header_length * 4) - 20)/4).ceil }, :onlyif => :is_options? do + sflow_ip4_addr :srcIpv4 + sflow_ip4_addr :dstIpv4 + array :ip_options, :initial_length => lambda { ip_header_length - 5 }, :onlyif => :is_options? do string :ip_option, :length => 4, :pad_byte => "\0" end - choice :ip_data, :selection => :ip_protocol, :onlyif => lambda { has_data?(size_header) } do + choice :ip_data, :selection => :ipProtocol, :onlyif => lambda { has_data?(size_header) } do tcp_header 6, :size_header => lambda { size_header - (ip_header_length * 4 * 8) } udp_header 17, :size_header => lambda { size_header - (ip_header_length * 4 * 8) } unknown_header :default, :size_header => lambda { size_header - (ip_header_length * 4 * 8) } @@ -87,7 +87,7 @@ class IPV4Header < BinData::Record def has_data?(size_header) bytes_left = size_header / 8 - ip_header_length * 4 - case ip_protocol + case ipProtocol when 6 return bytes_left >= 20 when 17 @@ -107,16 +107,16 @@ class IPV6Header < BinData::Record mandatory_parameter :size_header endian :big - bit4 :ip_version + bit4 :ipVersion bit6 :ip_dscp bit2 :ip_ecn bit20 :ipv6_flow_label uint16 :ip_payload_length - uint8 :ip_protocol + uint8 :protocol uint8 :ipv6_hop_limit - sflow_ip6_addr :src_ip - sflow_ip6_addr :dst_ip - choice :ip_data, :selection => :ip_protocol do + sflow_ip6_addr :srcIp + sflow_ip6_addr :dstIp + choice :ip_data, :selection => :protocol do tcp_header 6, :size_header => lambda { size_header - 320 } udp_header 17, :size_header => lambda { size_header - 320 } unknown_header :default, :size_header => lambda { size_header - 320 } @@ -130,7 +130,7 @@ class VLANHeader < BinData::Record endian :big bit3 :vlan_priority bit1 :vlan_cfi - bit12 :vlan_id + bit12 :vlanId uint16 :vlan_type choice :vlan_data, :selection => :vlan_type do ipv4_header 2048, :size_header => lambda { size_header - (4 * 8) } @@ -144,8 +144,8 @@ class EthernetHeader < BinData::Record mandatory_parameter :size_header endian :big - sflow_mac_address :eth_dst - sflow_mac_address :eth_src + sflow_mac_address :ethDst + sflow_mac_address :ethSrc uint16 :eth_type choice :eth_data, :selection => :eth_type do ipv4_header 2048, :size_header => lambda { size_header - (14 * 8) } diff --git a/lib/logstash/codecs/sflow/sample.rb b/lib/logstash/codecs/sflow/sample.rb index b4b5da8..5534110 100644 --- a/lib/logstash/codecs/sflow/sample.rb +++ b/lib/logstash/codecs/sflow/sample.rb @@ -74,12 +74,12 @@ class FlowSample < BinData::Record endian :big uint32 :flow_sequence_number uint8 :source_id_type - uint24 :source_id_index - uint32 :sampling_rate - uint32 :sample_pool + uint24 :sourceIdIndex + uint32 :samplingRate + uint32 :samplePool uint32 :drops - uint32 :input_interface - uint32 :output_interface + uint32 :input + uint32 :output uint32 :record_count array :records, :initial_length => :record_count do bit20 :record_entreprise @@ -96,7 +96,7 @@ class CounterSample < BinData::Record endian :big uint32 :sample_seq_number uint8 :source_id_type - uint24 :source_id_index + uint24 :sourceIdIndex uint32 :record_count array :records, :initial_length => :record_count do bit20 :record_entreprise @@ -113,15 +113,15 @@ class CounterSample < BinData::Record class ExpandedFlowSample < BinData::Record endian :big uint32 :flow_sequence_number - uint32 :source_id_type - uint32 :source_id_index - uint32 :sampling_rate - uint32 :sample_pool + uint32 :sourceIdType + uint32 :sourceIdIndex + uint32 :samplingRate + uint32 :samplePool uint32 :drops uint32 :input_interface_format - uint32 :input_interface_value + uint32 :input uint32 :output_interface_format - uint32 :output_interface_value + uint32 :output uint32 :record_count array :records, :initial_length => :record_count do bit20 :record_entreprise @@ -137,8 +137,8 @@ class ExpandedFlowSample < BinData::Record class ExpandedCounterSample < BinData::Record endian :big uint32 :sample_seq_number - uint32 :source_id_type - uint32 :source_id_index + uint32 :sourceIdType + uint32 :sourceIdIndex uint32 :record_count array :records, :initial_length => :record_count do bit20 :record_entreprise From 6442daa052b4b79266d82b4b25fe1cfa9c091d5d Mon Sep 17 00:00:00 2001 From: skytelan3 Date: Wed, 14 Aug 2019 13:45:13 +0900 Subject: [PATCH 2/4] =?UTF-8?q?packet,=20octets=20=EA=B3=84=EC=82=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/logstash/codecs/sflow.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/logstash/codecs/sflow.rb b/lib/logstash/codecs/sflow.rb index c145453..c6328cb 100644 --- a/lib/logstash/codecs/sflow.rb +++ b/lib/logstash/codecs/sflow.rb @@ -131,14 +131,14 @@ def decode(payload) assign_key_value(event, record) end - #compute octets - if event.include?('frame_length') and event.include?('sampling_rate') - event.set('octets', event.get('frame_length').to_i * event.get('sampling_rate').to_i) - end - + #@author jeonhn + #@change-date : 2019. 8. 13. #compute packets - if event.include?('packets') and event.include?('samplingRate') - event.set('packets', event.get('packets').to_i * event.get('samplingRate').to_i) + if event.include?('samplingRate') + event.set('packets', decoded['samples'].length * event.get('samplingRate').to_i) + if event.include?('frame_length') + event.set('octets', event.get('frame_length').to_i * event.get('samplingRate').to_i) + end end if sample['sample_format'] == 1 From c1a259f4be511282bb170971c730b4530cc6c9f7 Mon Sep 17 00:00:00 2001 From: skytelan3 Date: Wed, 14 Aug 2019 14:29:25 +0900 Subject: [PATCH 3/4] =?UTF-8?q?=EC=98=A4=EB=A5=98=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/logstash/codecs/sflow/packet_header.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/logstash/codecs/sflow/packet_header.rb b/lib/logstash/codecs/sflow/packet_header.rb index dc6b08d..166de95 100644 --- a/lib/logstash/codecs/sflow/packet_header.rb +++ b/lib/logstash/codecs/sflow/packet_header.rb @@ -79,7 +79,7 @@ class IPV4Header < BinData::Record array :ip_options, :initial_length => lambda { ip_header_length - 5 }, :onlyif => :is_options? do string :ip_option, :length => 4, :pad_byte => "\0" end - choice :ip_data, :selection => :ipProtocol, :onlyif => lambda { has_data?(size_header) } do + choice :ip_data, :selection => :protocol, :onlyif => lambda { has_data?(size_header) } do tcp_header 6, :size_header => lambda { size_header - (ip_header_length * 4 * 8) } udp_header 17, :size_header => lambda { size_header - (ip_header_length * 4 * 8) } unknown_header :default, :size_header => lambda { size_header - (ip_header_length * 4 * 8) } @@ -87,7 +87,7 @@ class IPV4Header < BinData::Record def has_data?(size_header) bytes_left = size_header / 8 - ip_header_length * 4 - case ipProtocol + case protocol when 6 return bytes_left >= 20 when 17 From e0c59000bd08b2ca9e886b22cfb6f1c8a76cf457 Mon Sep 17 00:00:00 2001 From: skytelan3 Date: Tue, 12 Sep 2023 16:37:58 +0900 Subject: [PATCH 4/4] =?UTF-8?q?=EC=99=80=EC=B9=98=ED=85=8D=EC=97=90=20?= =?UTF-8?q?=EC=95=8C=EB=A7=9E=EA=B2=8C=20=EA=B8=B0=EB=8A=A5=20=EB=B3=80?= =?UTF-8?q?=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/logstash/codecs/sflow.rb | 19 +++++++++++-------- lib/logstash/codecs/sflow/flow_record.rb | 5 ++++- lib/logstash/codecs/sflow/sample.rb | 10 +++++----- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/lib/logstash/codecs/sflow.rb b/lib/logstash/codecs/sflow.rb index 1ad79be..ea664db 100644 --- a/lib/logstash/codecs/sflow.rb +++ b/lib/logstash/codecs/sflow.rb @@ -136,14 +136,12 @@ def decode(payload) assign_key_value(event, record) end - #@author jeonhn - #@change-date : 2019. 8. 13. - #compute packets - if event.include?('samplingRate') - event.set('packets', decoded['samples'].length * event.get('samplingRate').to_i) - if event.include?('frame_length') - event.set('octets', event.get('frame_length').to_i * event.get('samplingRate').to_i) - end + #@author jeonhn + #@change-date : 2018. 7. 13. + #compute frame_length_times_sampling_rate + #packets to frame_length + if event.include?('packets') and event.include?('samplingRate') + event.set('octets', event.get('packets').to_i * event.get('samplingRate').to_i) end if sample['sample_format'] == 1 @@ -159,6 +157,11 @@ def decode(payload) #treat counter flow and expanded counter flow elsif sample['sample_entreprise'] == 0 && (sample['sample_format'] == 2 || sample['sample_format'] == 4) + #@author jeonhn + #@change-date : 2018. 7. 13. + #compute frame_length_times_sampling_rate + # continue + next sample['sample_data']['records'].each do |record| # Ensure that some data exist for the record if record['record_data'].to_s.eql? '' diff --git a/lib/logstash/codecs/sflow/flow_record.rb b/lib/logstash/codecs/sflow/flow_record.rb index 7278e5b..20de3d4 100644 --- a/lib/logstash/codecs/sflow/flow_record.rb +++ b/lib/logstash/codecs/sflow/flow_record.rb @@ -11,7 +11,10 @@ class RawPacketHeader < BinData::Buffer endian :big uint32 :headerProtocol - uint32 :frame_length + #@author jeonhn + #@change-date : 2018. 7. 13. + #@fix : packets to frame_length + uint32 :packets uint32 :stripped uint32 :header_size choice :sample_header, :selection => :headerProtocol do diff --git a/lib/logstash/codecs/sflow/sample.rb b/lib/logstash/codecs/sflow/sample.rb index fe368ef..c9a3786 100644 --- a/lib/logstash/codecs/sflow/sample.rb +++ b/lib/logstash/codecs/sflow/sample.rb @@ -77,10 +77,10 @@ class CounterSampleRecordData < BinData::Choice class FlowSample < BinData::Record endian :big uint32 :flow_sequence_number - uint8 :source_id_type + uint8 :sourceIdType uint24 :sourceIdIndex uint32 :samplingRate - uint32 :samplePool + uint32 :samplingPool uint32 :drops uint32 :input uint32 :output @@ -99,7 +99,7 @@ class FlowSample < BinData::Record class CounterSample < BinData::Record endian :big uint32 :sample_seq_number - uint8 :source_id_type + uint8 :sourceIdType uint24 :sourceIdIndex uint32 :record_count array :records, :initial_length => :record_count do @@ -122,9 +122,9 @@ class ExpandedFlowSample < BinData::Record uint32 :samplingRate uint32 :samplePool uint32 :drops - uint32 :input_interface_format + uint32 :inputInterfaceFormat uint32 :input - uint32 :output_interface_format + uint32 :outputInterfaceFormat uint32 :output uint32 :record_count array :records, :initial_length => :record_count do