I have follow code:
ProcessBuilder builder = new ProcessBuilder();
builder.command("sh", "-c", "cat eve.json | jq -c --arg s '2023-10-20T21:08:58.992978+0300' 'select(.timestamp | . >= $s)'");
builder.directory(new File("/Users/naysav/projectsIT/moshub/intrusion-detection-reactor/"));
Process process = builder.start();
InputStreamReader in = new InputStreamReader(process.getInputStream());
List<String> inList = new BufferedReader(in).lines().toList();
And file eve.json with follow lines:
{"timestamp":"2023-10-20T21:08:58.992978+0300","event_type":"stats","stats":{"uptime":87708,"capture":{"kernel_packets":439437,"kernel_drops":0,"errors":0,"afpacket":{"busy_loop_avg":0,"polls":428507,"poll_signal":0,"poll_timeout":386160,"poll_data":42347,"poll_errors":0,"send_errors":0}},"decoder":{"pkts":439437,"bytes":578328975,"invalid":0,"ipv4":436981,"ipv6":2061,"ethernet":439437,"arp":395,"unknown_ethertype":0,"chdlc":0,"raw":0,"null":0,"sll":0,"tcp":434317,"udp":4148,"sctp":0,"esp":0,"icmpv4":88,"icmpv6":487,"ppp":0,"pppoe":0,"geneve":0,"gre":0,"vlan":0,"vlan_qinq":0,"vlan_qinqinq":0,"vxlan":0,"vntag":0,"ieee8021ah":0,"teredo":0,"ipv4_in_ipv6":0,"ipv6_in_ipv6":0,"mpls":0,"avg_pkt_size":1316,"max_pkt_size":1514,"max_mac_addrs_src":0,"max_mac_addrs_dst":0,"erspan":0,"nsh":0,"event":{"ipv4":{"pkt_too_small":0,"hlen_too_small":0,"iplen_smaller_than_hlen":0,"trunc_pkt":0,"opt_invalid":0,"opt_invalid_len":0,"opt_malformed":0,"opt_pad_required":2,"opt_eol_required":0,"opt_duplicate":0,"opt_unknown":0,"wrong_ip_version":0,"icmpv6":0,"frag_pkt_too_large":0,"frag_overlap":0,"frag_ignored":0},"icmpv4":{"pkt_too_small":0,"unknown_type":0,"unknown_code":0,"ipv4_trunc_pkt":0,"ipv4_unknown_ver":0},"icmpv6":{"unknown_type":0,"unknown_code":0,"pkt_too_small":0,"ipv6_unknown_version":0,"ipv6_trunc_pkt":0,"mld_message_with_invalid_hl":0,"unassigned_type":0,"experimentation_type":0},"ipv6":{"pkt_too_small":0,"trunc_pkt":0,"trunc_exthdr":0,"exthdr_dupl_fh":0,"exthdr_useless_fh":0,"exthdr_dupl_rh":0,"exthdr_dupl_hh":0,"exthdr_dupl_dh":0,"exthdr_dupl_ah":0,"exthdr_dupl_eh":0,"exthdr_invalid_optlen":0,"wrong_ip_version":0,"exthdr_ah_res_not_null":0,"hopopts_unknown_opt":0,"hopopts_only_padding":0,"dstopts_unknown_opt":0,"dstopts_only_padding":0,"rh_type_0":0,"zero_len_padn":416,"fh_non_zero_reserved_field":0,"data_after_none_header":0,"unknown_next_header":0,"icmpv4":0,"frag_pkt_too_large":0,"frag_overlap":0,"frag_invalid_length":0,"frag_ignored":0,"ipv4_in_ipv6_too_small":0,"ipv4_in_ipv6_wrong_version":0,"ipv6_in_ipv6_too_small":0,"ipv6_in_ipv6_wrong_version":0},"tcp":{"pkt_too_small":0,"hlen_too_small":0,"invalid_optlen":0,"opt_invalid_len":0,"opt_duplicate":0},"udp":{"pkt_too_small":0,"hlen_too_small":0,"hlen_invalid":0,"len_invalid":0},"sll":{"pkt_too_small":0},"ethernet":{"pkt_too_small":0},"ppp":{"pkt_too_small":0,"vju_pkt_too_small":0,"ip4_pkt_too_small":0,"ip6_pkt_too_small":0,"wrong_type":0,"unsup_proto":0},"pppoe":{"pkt_too_small":0,"wrong_code":0,"malformed_tags":0},"gre":{"pkt_too_small":0,"wrong_version":0,"version0_recur":0,"version0_flags":0,"version0_hdr_too_big":0,"version0_malformed_sre_hdr":0,"version1_chksum":0,"version1_route":0,"version1_ssr":0,"version1_recur":0,"version1_flags":0,"version1_no_key":0,"version1_wrong_protocol":0,"version1_malformed_sre_hdr":0,"version1_hdr_too_big":0},"vlan":{"header_too_small":0,"unknown_type":0,"too_many_layers":0},"ieee8021ah":{"header_too_small":0},"vntag":{"header_too_small":0,"unknown_type":0},"ipraw":{"invalid_ip_version":0},"ltnull":{"pkt_too_small":0,"unsupported_type":0},"sctp":{"pkt_too_small":0},"esp":{"pkt_too_small":0},"mpls":{"header_too_small":0,"pkt_too_small":0,"bad_label_router_alert":0,"bad_label_implicit_null":0,"bad_label_reserved":0,"unknown_payload_type":0},"vxlan":{"unknown_payload_type":0},"geneve":{"unknown_payload_type":0},"erspan":{"header_too_small":0,"unsupported_version":0,"too_many_vlan_layers":0},"dce":{"pkt_too_small":0},"chdlc":{"pkt_too_small":0},"nsh":{"header_too_small":0,"unsupported_version":0,"bad_header_length":0,"reserved_type":0,"unsupported_type":0,"unknown_payload":0}},"too_many_layers":0},"tcp":{"syn":101,"synack":94,"rst":8,"active_sessions":2,"sessions":96,"ssn_memcap_drop":0,"ssn_from_cache":15,"ssn_from_pool":81,"pseudo":0,"pseudo_failed":0,"invalid_checksum":0,"midstream_pickups":0,"pkt_on_wrong_thread":0,"ack_unseen_data":0,"segment_memcap_drop":0,"segment_from_cache":10973,"segment_from_pool":284,"stream_depth_reached":13,"reassembly_gap":0,"overlap":0,"overlap_diff_data":0,"insert_data_normal_fail":0,"insert_data_overlap_fail":0,"memuse":1212416,"reassembly_memuse":237568},"flow":{"memcap":0,"total":882,"active":9,"tcp":96,"udp":625,"icmpv4":0,"icmpv6":161,"tcp_reuse":0,"get_used":0,"get_used_eval":0,"get_used_eval_reject":0,"get_used_eval_busy":0,"get_used_failed":0,"wrk":{"spare_sync_avg":99,"spare_sync":9,"spare_sync_incomplete":1,"spare_sync_empty":0,"flows_evicted_needs_work":19,"flows_evicted_pkt_inject":21,"flows_evicted":100,"flows_injected":19,"flows_injected_max":2},"end":{"state":{"new":509,"established":272,"closed":92,"local_bypassed":0,"capture_bypassed":0},"tcp_state":{"none":0,"syn_sent":2,"syn_recv":0,"established":0,"fin_wait1":0,"fin_wait2":0,"time_wait":0,"last_ack":0,"close_wait":0,"closing":0,"closed":92},"tcp_liberal":0},"mgr":{"full_hash_pass":1970,"rows_per_sec":6553,"rows_maxlen":2,"flows_checked":1833,"flows_notimeout":1058,"flows_timeout":775,"flows_evicted":775,"flows_evicted_needs_work":19},"spare":9857,"emerg_mode_entered":0,"emerg_mode_over":0,"recycler":{"recycled":756,"queue_avg":0,"queue_max":3},"memuse":11428608},"defrag":{"ipv4":{"fragments":0,"reassembled":0},"ipv6":{"fragments":0,"reassembled":0},"max_frag_hits":0},"flow_bypassed":{"local_pkts":0,"local_bytes":0,"local_capture_pkts":0,"local_capture_bytes":0,"closed":0,"pkts":0,"bytes":0},"detect":{"engines":[{"id":0,"last_reload":"2023-10-19T20:47:10.316815+0300","rules_loaded":2,"rules_failed":0}],"alert":4,"alert_queue_overflow":0,"alerts_suppressed":0},"app_layer":{"flow":{"http":84,"ftp":0,"smtp":0,"tls":8,"ssh":0,"imap":0,"smb":1,"dcerpc_tcp":0,"dns_tcp":1,"nfs_tcp":0,"ntp":70,"ftp-data":0,"tftp":0,"ike":0,"krb5_tcp":0,"quic":0,"dhcp":70,"snmp":0,"sip":0,"rfb":0,"mqtt":0,"telnet":0,"rdp":0,"http2":0,"bittorrent-dht":0,"failed_tcp":0,"dcerpc_udp":0,"dns_udp":197,"nfs_udp":0,"krb5_udp":0,"failed_udp":288},"tx":{"http":305,"ftp":0,"smtp":0,"tls":0,"ssh":0,"imap":0,"smb":1,"dcerpc_tcp":0,"dns_tcp":4,"nfs_tcp":0,"ntp":74,"ftp-data":0,"tftp":0,"ike":0,"krb5_tcp":0,"quic":0,"dhcp":213,"snmp":0,"sip":0,"rfb":0,"mqtt":0,"telnet":0,"rdp":0,"http2":0,"bittorrent-dht":0,"dcerpc_udp":0,"dns_udp":403,"nfs_udp":0,"krb5_udp":0},"error":{"http":{"gap":0,"alloc":0,"parser":0,"internal":0},"ftp":{"gap":0,"alloc":0,"parser":0,"internal":0},"smtp":{"gap":0,"alloc":0,"parser":0,"internal":0},"tls":{"gap":0,"alloc":0,"parser":0,"internal":0},"ssh":{"gap":0,"alloc":0,"parser":0,"internal":0},"imap":{"gap":0,"alloc":0,"parser":0,"internal":0},"smb":{"gap":0,"alloc":0,"parser":0,"internal":0},"dcerpc_tcp":{"gap":0,"alloc":0,"parser":0,"internal":0},"dns_tcp":{"gap":0,"alloc":0,"parser":0,"internal":0},"nfs_tcp":{"gap":0,"alloc":0,"parser":0,"internal":0},"ntp":{"gap":0,"alloc":0,"parser":0,"internal":0},"ftp-data":{"gap":0,"alloc":0,"parser":0,"internal":0},"tftp":{"gap":0,"alloc":0,"parser":0,"internal":0},"ike":{"gap":0,"alloc":0,"parser":0,"internal":0},"krb5_tcp":{"gap":0,"alloc":0,"parser":0,"internal":0},"quic":{"gap":0,"alloc":0,"parser":0,"internal":0},"dhcp":{"gap":0,"alloc":0,"parser":0,"internal":0},"snmp":{"gap":0,"alloc":0,"parser":0,"internal":0},"sip":{"gap":0,"alloc":0,"parser":0,"internal":0},"rfb":{"gap":0,"alloc":0,"parser":0,"internal":0},"mqtt":{"gap":0,"alloc":0,"parser":0,"internal":0},"telnet":{"gap":0,"alloc":0,"parser":0,"internal":0},"rdp":{"gap":0,"alloc":0,"parser":0,"internal":0},"http2":{"gap":0,"alloc":0,"parser":0,"internal":0},"bittorrent-dht":{"gap":0,"alloc":0,"parser":0,"internal":0},"failed_tcp":{"gap":0},"dcerpc_udp":{"alloc":0,"parser":0,"internal":0},"dns_udp":{"alloc":0,"parser":0,"internal":0},"nfs_udp":{"alloc":0,"parser":0,"internal":0},"krb5_udp":{"alloc":0,"parser":0,"internal":0}},"expectations":0},"memcap_pressure":8,"memcap_pressure_max":8,"http":{"memuse":208,"memcap":0},"ftp":{"memuse":0,"memcap":0},"file_store":{"open_files":0}}}
...
Environment: Java 17, Mac OS (Ventura 13), Spring Boot 3.1.1
My goal is parsing json-logs from file from some date.
For this I try using jq, but when it run by java result is empty list.
Running by zsh terminal returns expected result:
`cat eve.json | jq -c --arg s '2023-10-20T21:08:15.000153+0300'
{"timestamp":"2023-10-20T21:08:18.976750+0300","event_type":"stats","stats":{"uptime":87668,"capture":{"kernel_packets":439437,"kernel_drops":0,"errors":0,"afpacket":{"busy_loop_avg":0,"polls":427733,"poll_signal":0,"poll_timeout":385386,"poll_data":42347,"poll_errors":0,"send_errors":0}},"decoder":{"pkts":439437,"bytes":578328975,"invalid":0,"ipv4":436981,"ipv6":2061,"ethernet":439437,"arp":395,"unknown_ethertype":0,"chdlc":0,"raw":0,"null":0,"sll":0,"tcp":434317,"udp":4148,"sctp":0,"esp":0,"icmpv4":88,"icmpv6":487,"ppp":0,"pppoe":0,"geneve":0,"gre":0,"vlan":0,"vlan_qinq":0,"vlan_qinqinq":0,"vxlan":0,"vntag":0,"ieee8021ah":0,"teredo":0,"ipv4_in_ipv6":0,"ipv6_in_ipv6":0,"mpls":0,"avg_pkt_size":1316,"max_pkt_size":1514,"max_mac_addrs_src":0,"max_mac_addrs_dst":0,"erspan":0,"nsh":0,"event":{"ipv4":{"pkt_too_small":0,"hlen_too_small":0,"iplen_smaller_than_hlen":0,"trunc_pkt":0,"opt_invalid":0,"opt_invalid_len":0,"opt_malformed":0,"opt_pad_required":2,"opt_eol_required":0,"opt_duplicate":0,"opt_unknown":0,"wrong_ip_version":0,"icmpv6":0,"frag_pkt_too_large":0,"frag_overlap":0,"frag_ignored":0},"icmpv4":{"pkt_too_small":0,"unknown_type":0,"unknown_code":0,"ipv4_trunc_pkt":0,"ipv4_unknown_ver":0},"icmpv6":{"unknown_type":0,"unknown_code":0,"pkt_too_small":0,"ipv6_unknown_version":0,"ipv6_trunc_pkt":0,"mld_message_with_invalid_hl":0,"unassigned_type":0,"experimentation_type":0},"ipv6":{"pkt_too_small":0,"trunc_pkt":0,"trunc_exthdr":0,"exthdr_dupl_fh":0,"exthdr_useless_fh":0,"exthdr_dupl_rh":0,"exthdr_dupl_hh":0,"exthdr_dupl_dh":0,"exthdr_dupl_ah":0,"exthdr_dupl_eh":0,"exthdr_invalid_optlen":0,"wrong_ip_version":0,"exthdr_ah_res_not_null":0,"hopopts_unknown_opt":0,"hopopts_only_padding":0,"dstopts_unknown_opt":0,"dstopts_only_padding":0,"rh_type_0":0,"zero_len_padn":416,"fh_non_zero_reserved_field":0,"data_after_none_header":0,"unknown_next_header":0,"icmpv4":0,"frag_pkt_too_large":0,"frag_overlap":0,"frag_invalid_length":0,"frag_ignored":0,"ipv4_in_ipv6_too_small":0,"ipv4_in_ipv6_wrong_version":0,"ipv6_in_ipv6_too_small":0,"ipv6_in_ipv6_wrong_version":0},"tcp":{"pkt_too_small":0,"hlen_too_small":0,"invalid_optlen":0,"opt_invalid_len":0,"opt_duplicate":0},"udp":{"pkt_too_small":0,"hlen_too_small":0,"hlen_invalid":0,"len_invalid":0},"sll":{"pkt_too_small":0},"ethernet":{"pkt_too_small":0},"ppp":{"pkt_too_small":0,"vju_pkt_too_small":0,"ip4_pkt_too_small":0,"ip6_pkt_too_small":0,"wrong_type":0,"unsup_proto":0},"pppoe":{"pkt_too_small":0,"wrong_code":0,"malformed_tags":0},"gre":{"pkt_too_small":0,"wrong_version":0,"version0_recur":0,"version0_flags":0,"version0_hdr_too_big":0,"version0_malformed_sre_hdr":0,"version1_chksum":0,"version1_route":0,"version1_ssr":0,"version1_recur":0,"version1_flags":0,"version1_no_key":0,"version1_wrong_protocol":0,"version1_malformed_sre_hdr":0,"version1_hdr_too_big":0},"vlan":{"header_too_small":0,"unknown_type":0,"too_many_layers":0},"ieee8021ah":{"header_too_small":0},"vntag":{"header_too_small":0,"unknown_type":0},"ipraw":{"invalid_ip_version":0},"ltnull":{"pkt_too_small":0,"unsupported_type":0},"sctp":{"pkt_too_small":0},"esp":{"pkt_too_small":0},"mpls":{"header_too_small":0,"pkt_too_small":0,"bad_label_router_alert":0,"bad_label_implicit_null":0,"bad_label_reserved":0,"unknown_payload_type":0},"vxlan":{"unknown_payload_type":0},"geneve":{"unknown_payload_type":0},"erspan":{"header_too_small":0,"unsupported_version":0,"too_many_vlan_layers":0},"dce":{"pkt_too_small":0},"chdlc":{"pkt_too_small":0},"nsh":{"header_too_small":0,"unsupported_version":0,"bad_header_length":0,"reserved_type":0,"unsupported_type":0,"unknown_payload":0}},"too_many_layers":0},"tcp":{"syn":101,"synack":94,"rst":8,"active_sessions":2,"sessions":96,"ssn_memcap_drop":0,"ssn_from_cache":15,"ssn_from_pool":81,"pseudo":0,"pseudo_failed":0,"invalid_checksum":0,"midstream_pickups":0,"pkt_on_wrong_thread":0,"ack_unseen_data":0,"segment_memcap_drop":0,"segment_from_cache":10973,"segment_from_pool":284,"stream_depth_reached":13,"reassembly_gap":0,"overlap":0,"overlap_diff_data":0,"insert_data_normal_fail":0,"insert_data_overlap_fail":0,"memuse":1212416,"reassembly_memuse":237568},"flow":{"memcap":0,"total":882,"active":9,"tcp":96,"udp":625,"icmpv4":0,"icmpv6":161,"tcp_reuse":0,"get_used":0,"get_used_eval":0,"get_used_eval_reject":0,"get_used_eval_busy":0,"get_used_failed":0,"wrk":{"spare_sync_avg":99,"spare_sync":9,"spare_sync_incomplete":1,"spare_sync_empty":0,"flows_evicted_needs_work":19,"flows_evicted_pkt_inject":21,"flows_evicted":100,"flows_injected":19,"flows_injected_max":2},"end":{"state":{"new":509,"established":272,"closed":92,"local_bypassed":0,"capture_bypassed":0},"tcp_state":{"none":0,"syn_sent":2,"syn_recv":0,"established":0,"fin_wait1":0,"fin_wait2":0,"time_wait":0,"last_ack":0,"close_wait":0,"closing":0,"closed":92},"tcp_liberal":0},"mgr":{"full_hash_pass":1966,"rows_per_sec":6553,"rows_maxlen":2,"flows_checked":1832,"flows_notimeout":1057,"flows_timeout":775,"flows_evicted":775,"flows_evicted_needs_work":19},"spare":9857,"emerg_mode_entered":0,"emerg_mode_over":0,"recycler":{"recycled":756,"queue_avg":0,"queue_max":3},"memuse":11428608},"defrag":{"ipv4":{"fragments":0,"reassembled":0},"ipv6":{"fragments":0,"reassembled":0},"max_frag_hits":0},"flow_bypassed":{"local_pkts":0,"local_bytes":0,"local_capture_pkts":0,"local_capture_bytes":0,"closed":0,"pkts":0,"bytes":0},"detect":{"engines":[{"id":0,"last_reload":"2023-10-19T20:47:10.316815+0300","rules_loaded":2,"rules_failed":0}],"alert":4,"alert_queue_overflow":0,"alerts_suppressed":0},"app_layer":{"flow":{"http":84,"ftp":0,"smtp":0,"tls":8,"ssh":0,"imap":0,"smb":1,"dcerpc_tcp":0,"dns_tcp":1,"nfs_tcp":0,"ntp":70,"ftp-data":0,"tftp":0,"ike":0,"krb5_tcp":0,"quic":0,"dhcp":70,"snmp":0,"sip":0,"rfb":0,"mqtt":0,"telnet":0,"rdp":0,"http2":0,"bittorrent-dht":0,"failed_tcp":0,"dcerpc_udp":0,"dns_udp":197,"nfs_udp":0,"krb5_udp":0,"failed_udp":288},"tx":{"http":305,"ftp":0,"smtp":0,"tls":0,"ssh":0,"imap":0,"smb":1,"dcerpc_tcp":0,"dns_tcp":4,"nfs_tcp":0,"ntp":74,"ftp-data":0,"tftp":0,"ike":0,"krb5_tcp":0,"quic":0,"dhcp":213,"snmp":0,"sip":0,"rfb":0,"mqtt":0,"telnet":0,"rdp":0,"http2":0,"bittorrent-dht":0,"dcerpc_udp":0,"dns_udp":403,"nfs_udp":0,"krb5_udp":0},"error":{"http":{"gap":0,"alloc":0,"parser":0,"internal":0},"ftp":{"gap":0,"alloc":0,"parser":0,"internal":0},"smtp":{"gap":0,"alloc":0,"parser":0,"internal":0},"tls":{"gap":0,"alloc":0,"parser":0,"internal":0},"ssh":{"gap":0,"alloc":0,"parser":0,"internal":0},"imap":{"gap":0,"alloc":0,"parser":0,"internal":0},"smb":{"gap":0,"alloc":0,"parser":0,"internal":0},"dcerpc_tcp":{"gap":0,"alloc":0,"parser":0,"internal":0},"dns_tcp":{"gap":0,"alloc":0,"parser":0,"internal":0},"nfs_tcp":{"gap":0,"alloc":0,"parser":0,"internal":0},"ntp":{"gap":0,"alloc":0,"parser":0,"internal":0},"ftp-data":{"gap":0,"alloc":0,"parser":0,"internal":0},"tftp":{"gap":0,"alloc":0,"parser":0,"internal":0},"ike":{"gap":0,"alloc":0,"parser":0,"internal":0},"krb5_tcp":{"gap":0,"alloc":0,"parser":0,"internal":0},"quic":{"gap":0,"alloc":0,"parser":0,"internal":0},"dhcp":{"gap":0,"alloc":0,"parser":0,"internal":0},"snmp":{"gap":0,"alloc":0,"parser":0,"internal":0},"sip":{"gap":0,"alloc":0,"parser":0,"internal":0},"rfb":{"gap":0,"alloc":0,"parser":0,"internal":0},"mqtt":{"gap":0,"alloc":0,"parser":0,"internal":0},"telnet":{"gap":0,"alloc":0,"parser":0,"internal":0},"rdp":{"gap":0,"alloc":0,"parser":0,"internal":0},"http2":{"gap":0,"alloc":0,"parser":0,"internal":0},"bittorrent-dht":{"gap":0,"alloc":0,"parser":0,"internal":0},"failed_tcp":{"gap":0},"dcerpc_udp":{"alloc":0,"parser":0,"internal":0},"dns_udp":{"alloc":0,"parser":0,"internal":0},"nfs_udp":{"alloc":0,"parser":0,"internal":0},"krb5_udp":{"alloc":0,"parser":0,"internal":0}},"expectations":0},"memcap_pressure":8,"memcap_pressure_max":8,"http":{"memuse":208,"memcap":0},"ftp":{"memuse":0,"memcap":0},"file_store":{"open_files":0}}}
...
Also ProcessBuilder returns expected result when using grep:
builder.command("sh", "-c", "cat eve.json | grep '2023-10-20T21:08:58.992978+0300'");
If I run the cat eve.json | jq -c --arg s '2023-10-20T21:08:58.992978+0300' 'select(.timestamp | . >= $s) through a script.sh by ProcessBuilder, the result is the same.
What am I missing?
Make sure that jq can be found in the PATH environment variable of your process or use the absolute path to the jq binary (e.g. /usr/bin/jq).
That said, your process builder is unnecessarily complicated and forks 3 processes when a single one suffices:
builder.command(
"jq", // or the absolute path
"-c",
"--arg",
"s",
"2023-10-20T21:08:58.992978+0300",
"select(.timestamp | . >= $s)");
This also avoids any shell quoting issues.