/// trp.proto - Trisul Remote Protocol .proto file /// TRP : Trisul Remote Protocol is a remote query API that allows /// clients to connect and retrieve data from Trisul Hub // Trisul Remote Protocol (TRP) definition // Based on Google Protocol Buffers // (c) 2012-16, Unleash Networks (http://www.unleashnetworks.com) // $Rev: 6946 $ option optimize_for = LITE_RUNTIME; package TRP; // // Basic structures // /// Timestamp : Epoch time unix time (seconds since Jan 1 1970) message Timestamp { required int64 tv_sec=1; optional int64 tv_usec=2 [default=0]; } /// TimeInterval from and to message TimeInterval { required Timestamp from=1; /// start time required Timestamp to=2; /// end time } /// StatsTuple : a single timeseries vaue (t,v) message StatsTuple { required Timestamp ts=1; /// ts required int64 val=2; /// value metric } /// StatsArray : multiple timeseries values (t, v1, v2, v3...vn) /// notice we use ts_tv_sec. Most Trisul data have 1 sec resolution. message StatsArray { required int64 ts_tv_sec=1; /// tv.tv_sec repeated int64 values=2; /// array of values } /// MeterValues : a timeseries (meter_id, stat1, stat2, ... statn) /// this is rarely used because StatsArray is available . message MeterValues { required int32 meter=1; /// metric id , eg Hosts:TotalConnections repeated StatsTuple values=2; optional int64 total=3; /// total of all metric values optional int64 seconds=4; /// total number of seconds in time series } /// MeterType : information about a particular meter /// message MeterInfo { /// types of meters // from TrisulAPI enum MeterType { VT_INVALID=0; VT_RATE_COUNTER_WITH_SLIDING_WINDOW=1;/// this for top-N type counters VT_COUNTER=2; /// basic counter, stores val in the raw VT_COUNTER_WITH_SLIDING_WINDOW=3; /// use this for top-N type counters VT_RATE_COUNTER=4; /// rate counter stores val/sec VT_GAUGE=5; /// basic gauge VT_GAUGE_MIN_MAX_AVG=6; /// gauge with 3 additional min/avg/max cols (auto) VT_AUTO=7; /// automatic (eg, min/max/avg/stddev/) VT_RUNNING_COUNTER=8; /// running counter, no delta calc VT_AVERAGE=9; /// average of samples, total/sampl uses 32bt|32bit VT_DELTA_RATE_COUNTER=10; /// link snmp running counter VT_MAX=11; /// max of samples VT_MIN=12; /// min of samples } required int32 id=1; required MeterType type=2; required int32 topcount=3; required string name=4; optional string description=5; optional string units=6; } /// KeyStats - A full time series item (countergroup, key, timeseries) /// message KeyStats { required string counter_group=2; /// guid of counter group required KeyT key=3; /// key representing an item repeated MeterValues meters=4; /// array of timeseries (timeseries-meter0, ts-meter1, ...ts-meter-n) } /// KeyT : Represents a Key /// Top level objects are named ObjT /// eg KeyT - Key Type, SessionT - Session Type etc. message KeyT { optional string key=1; /// key in trisul key format eg, C0.A8.01.02 for 192.168.1.2 optional string readable=2; /// human friendly name optional string label=3; /// a user label eg, a hostname or manually assigned name optional string description=4; /// description optional int64 metric=5; /// optional : a single metric value - relevant to the query used message NameValueT { required string attr_name=1; required string attr_value=2; } repeated NameValueT attributes=6; /// New: Attributes are NVP for keys optional int64 metric_max=7; /// optional : max observed value for metric in context optional int64 metric_min=8; /// optional : max observed value for metric optional int64 metric_avg=9; /// optional : max observed value for metric } /// CounterGroupT : Represents a counter group /// message CounterGroupT { required string guid=1; /// guid identifying the CG required string name=2; /// CG name optional int64 bucket_size=3; /// bucketsize for all meters in this group optional TimeInterval time_interval=4; /// total time interval available in DB optional int64 topper_bucket_size=5; /// topper bucketsize (streaming analytics window) repeated MeterInfo meters=6; /// array of meter information (m0, m1, .. mn) message Crosskey { required string parentguid=1; /// parent GUID of ckey counter group required string crosskeyguid_1=2; /// crosskey 1 guid - required optional string crosskeyguid_2=3; /// crosskey 2 guid - optional } optional Crosskey crosskey=7; /// for cross key counter group } /// SessionT : an IP flow /// message SessionT { optional string session_key=1; /// Trisul format eg 06A:C0.A8.01.02:p-0B94_D1.D8.F9.3A:p-0016 required string session_id=2; /// SID once stored in DB 883:3:883488 optional string user_label=3; /// any label assigned by user required TimeInterval time_interval=4; /// start and end time of flow optional int64 state=5; /// flow state (see docs) optional int64 az_bytes=6; /// bytes in A>Z direction, see KeyA>KeyZ optional int64 za_bytes=7; /// bytes in Z>A direction optional int64 az_packets=8; /// pkts in A>Z direction optional int64 za_packets=9; /// pkts in Z>A direction required KeyT key1A=10; /// basically IP A End required KeyT key2A=11; /// Port Z End (can be a string like ICMP00, GRE00, for non TCP/UDP) required KeyT key1Z=12; /// IP Z end required KeyT key2Z=13; /// Port Z End required KeyT protocol=14; /// IP Protocol optional KeyT nf_routerid=15; /// Netflow only : Router ID optional KeyT nf_ifindex_in=16; /// Netflow only : Interface Index optional KeyT nf_ifindex_out=17; /// Netflow only : Interface Index optional string tags=18; /// tags assigned using flow taggers optional int64 az_payload=19; /// AZ payload - actual content transferred optional int64 za_payload=20; /// ZA payload optional int64 setup_rtt=21; /// Round Trip Time for setup : Must have TCPReassmbly enabled on Probe optional int64 retransmissions=22; /// Retransmissiosn total optional int64 tracker_statval=23; /// Metric for flow trackers optional string probe_id=24; /// Probe ID generating this flow } /// AlertT : an alert in Trisul /// all alert types Threshold Crossing, Flow Tracker, Badfellas, custom alerts use /// the same object below message AlertT{ optional int64 sensor_id=1; /// source of alert, usually not used required Timestamp time=2; /// timestamp required string alert_id=3; /// DB alert ID eg 99:8:98838 optional KeyT source_ip=4; /// source ip optional KeyT source_port=5; optional KeyT destination_ip=6; optional KeyT destination_port=7; optional KeyT sigid=8; /// unique key representing alert type optional KeyT classification=9; /// classification (from IDS terminology) optional KeyT priority=10; /// priority 1,2,3 optional Timestamp dispatch_time=11; /// sent time optional string dispatch_message1=12; /// a free format string created by generator of alert optional string dispatch_message2=13; /// second format optional int64 occurrances=14[default=1];/// number of occurranes, used by QueryAlerts for aggregation optional string group_by_key=15; /// aggregation key optional string probe_id=16; /// probe generating this alert optional string alert_status=17; /// FIRE,CLEAR,BLOCK etc optional int64 acknowledge_flag=18; /// ACK or NOT } /// ResourceT : represents a "resource" object /// examples DNS records, HTTP URLs, TLS Certificates, extracted file hashes, etc message ResourceT { required Timestamp time=1; /// time resource was seen required string resource_id=2; /// DB id format = 988:0:8388383 optional KeyT source_ip=3; optional KeyT source_port=4; optional KeyT destination_ip=5; optional KeyT destination_port=6; optional string uri=7; /// raw resource - uniform resource id ,dns names, http url, etc optional string userlabel=8; /// additional data optional string probe_id=9; /// which probe detected this } /// DocumentT : a full text document /// full HTTP headers, printable TLS certs, etc message DocumentT { required string dockey=1; /// unique id optional string fts_attributes=2; /// attibutes used for facets optional string fullcontent=3; /// full document text /// this document was seen at these time and on this flow message Flow { required Timestamp time=1; required string key=2; } repeated Flow flows=4; /// list of flows where this doc was seen optional string probe_id=5; } /// VertexGroupT : a group of vertices /// message VertexGroupT { required string vertex_group=1; /// GUID of vertices in this message repeated KeyT vertex_keys=2; /// list of vertices } /// EdgeGraphT : a graph /// subjectnode -> vertices(of a particular type) message EdgeGraphT { required TimeInterval time_interval=4; /// covers this window repeated VertexGroupT vertex_groups=3; /// vertices grouped by type } /// Enums /// Auth Level enum AuthLevel { ADMIN=1; BASIC_USER=2; FORENSIC_USER=3; BLOCKED_USER=4; } /// Compression: Used by PCAP or other content requests enum CompressionType { UNCOMPRESSED=1; GZIP=2; } /// Pcap: format enum PcapFormat { LIBPCAP=1; /// normal libpcap format *.pcap UNSNIFF=2; /// LIBPCAPNOFILEHEADER=3; /// libpcap but without the pcap file header } enum DomainNodeType { HUB=0; PROBE=1; CONFIG=2; ROUTER=3; WEB=4; MONITOR=5; } enum DomainOperation { GETNODES=1; HEARTBEAT=2; REGISTER=3; } message NameValue { required string name=1; optional string value=2; } /// Top level message is TRP::Message /// wraps the actual request or response /// /// You must set trp.command = for EACH request in addition to /// constructing the actual TRP request message /// message Message { enum Command { HELLO_REQUEST=1; HELLO_RESPONSE=2; OK_RESPONSE=3; ERROR_RESPONSE=5; COUNTER_GROUP_TOPPER_REQUEST=6; COUNTER_GROUP_TOPPER_RESPONSE=7; COUNTER_ITEM_REQUEST=8; COUNTER_ITEM_RESPONSE=9; PCAP_REQUEST=14; PCAP_RESPONSE=15; SEARCH_KEYS_REQUEST=18; SEARCH_KEYS_RESPONSE=19; COUNTER_GROUP_INFO_REQUEST=20; COUNTER_GROUP_INFO_RESPONSE=21; SESSION_TRACKER_REQUEST=22; SESSION_TRACKER_RESPONSE=23; UPDATE_KEY_REQUEST=32; UPDATE_KEY_RESPONSE=33; QUERY_SESSIONS_REQUEST=34; QUERY_SESSIONS_RESPONSE=35; PROBE_STATS_REQUEST=38; PROBE_STATS_RESPONSE=39; QUERY_ALERTS_REQUEST=44; QUERY_ALERTS_RESPONSE=45; QUERY_RESOURCES_REQUEST=48; QUERY_RESOURCES_RESPONSE=49; GREP_REQUEST=60; GREP_RESPONSE=61; KEYSPACE_REQUEST=70; KEYSPACE_RESPONSE=71; TOPPER_TREND_REQUEST=72; TOPPER_TREND_RESPONSE=73; STAB_PUBSUB_CTL=80; QUERY_FTS_REQUEST=90; QUERY_FTS_RESPONSE=91; TIMESLICES_REQUEST=92; TIMESLICES_RESPONSE=93; DELETE_ALERTS_REQUEST=94; METRICS_SUMMARY_REQUEST=95; METRICS_SUMMARY_RESPONSE=96; PCAP_SLICES_REQUEST=97; SERVICE_REQUEST=101; SERVICE_RESPONSE=102; CONFIG_REQUEST=103; CONFIG_RESPONSE=104; LOG_REQUEST=105; LOG_RESPONSE=106; CONTEXT_CREATE_REQUEST=108; CONTEXT_DELETE_REQUEST=109; CONTEXT_START_REQUEST=110; CONTEXT_STOP_REQUEST=111; CONTEXT_INFO_REQUEST=112; CONTEXT_INFO_RESPONSE=113; CONTEXT_CONFIG_REQUEST=114; CONTEXT_CONFIG_RESPONSE=115; DOMAIN_REQUEST=116; DOMAIN_RESPONSE=117; NODE_CONFIG_REQUEST=118; NODE_CONFIG_RESPONSE=119; ASYNC_REQUEST=120; ASYNC_RESPONSE=121; FILE_REQUEST=122; FILE_RESPONSE=123; SUBSYSTEM_INIT=124; // init msg used to prepare services SUBSYSTEM_EXIT=125; GRAPH_REQUEST=130; GRAPH_RESPONSE=131; AGGREGATE_SESSIONS_REQUEST=132; AGGREGATE_SESSIONS_RESPONSE=133; RUNTOOL_REQUEST=134; RUNTOOL_RESPONSE=135; HA_CONTROL_REQUEST=140; HA_CONTROL_RESPONSE=141; DOMAIN_PING=142; TOOL_INFO_REQUEST=143; TOOL_INFO_RESPONSE=144; } required Command trp_command=1; optional HelloRequest hello_request=2; optional HelloResponse hello_response=3; optional OKResponse ok_response=4; optional ErrorResponse error_response=5; optional CounterGroupTopperRequest counter_group_topper_request=6; optional CounterGroupTopperResponse counter_group_topper_response=7; optional CounterItemRequest counter_item_request=8; optional CounterItemResponse counter_item_response=9; optional PcapRequest pcap_request=14; optional PcapResponse pcap_response=15; optional SearchKeysRequest search_keys_request=18; optional SearchKeysResponse search_keys_response=19; optional CounterGroupInfoRequest counter_group_info_request=20; optional CounterGroupInfoResponse counter_group_info_response=21; optional UpdateKeyRequest update_key_request=30; optional QuerySessionsRequest query_sessions_request=31; optional QuerySessionsResponse query_sessions_response=32; optional SessionTrackerRequest session_tracker_request=33; optional SessionTrackerResponse session_tracker_response=34; optional ProbeStatsRequest probe_stats_request=37; optional ProbeStatsResponse probe_stats_response=38; optional QueryAlertsRequest query_alerts_request=43; optional QueryAlertsResponse query_alerts_response=44; optional QueryResourcesRequest query_resources_request=47; optional QueryResourcesResponse query_resources_response=48; optional GrepRequest grep_request=51; optional GrepResponse grep_response=52; optional TopperTrendRequest topper_trend_request=55; optional TopperTrendResponse topper_trend_response=56; optional SubscribeCtl subscribe_ctl=59; optional QueryFTSRequest query_fts_request=60; optional QueryFTSResponse query_fts_response=61; optional TimeSlicesRequest time_slices_request=62; optional TimeSlicesResponse time_slices_response=63; optional DeleteAlertsRequest delete_alerts_request=64; optional MetricsSummaryRequest metrics_summary_request=65; optional MetricsSummaryResponse metrics_summary_response=66; optional KeySpaceRequest key_space_request=67; optional KeySpaceResponse key_space_response=68; optional PcapSlicesRequest pcap_slices_request=69; optional LogRequest log_request=105; optional LogResponse log_response=106; optional ContextCreateRequest context_create_request=108; optional ContextDeleteRequest context_delete_request=109; optional ContextStartRequest context_start_request=110; optional ContextStopRequest context_stop_request=111; optional ContextConfigRequest context_config_request=112; optional ContextConfigResponse context_config_response=113; optional ContextInfoRequest context_info_request=114; optional ContextInfoResponse context_info_response=115; optional DomainRequest domain_request=116; optional DomainResponse domain_response=117; optional NodeConfigRequest node_config_request=118; optional NodeConfigResponse node_config_response=119; optional AsyncRequest async_request=120; optional AsyncResponse async_response=121; optional FileRequest file_request=122; optional FileResponse file_response=123; optional GraphRequest graph_request=130; optional GraphResponse graph_response=131; optional AggregateSessionsRequest aggregate_sessions_request=140; optional AggregateSessionsResponse aggregate_sessions_response=141; optional AggregateResourcesRequest aggregate_resources_request=142; optional AggregateResourcesResponse aggregate_resources_response=143; optional RunToolRequest run_tool_request=144; optional RunToolResponse run_tool_response=145; optional HAControlRequest ha_control_request=150; optional HAControlResponse ha_control_response=151; optional ToolInfoRequest tool_info_request=152; optional ToolInfoResponse tool_info_response=153; optional string destination_node=200; // todo move 2nd optional string probe_id=201; // todo move 3rd optional bool run_async=202; /// if run_async = true, then you will immediately get a AsynResponse with a token you can poll } ////////////////////////////////////////////////////////////// // --------------- Messages Section -------------------------// // TRP is a synchronous request response protocol // // Send a xyz_Request() and wait for xyz_Response // // All requests necessarily get a response // // --------------- Messages Section -------------------------// ////////////////////////////////////////////////////////////// /// Hello Request : use to check connectivity message HelloRequest{ required string station_id=1; /// an id of the query client trying to connect optional string message=2; /// a message (will be echoed back in response) } message HelloResponse{ required string station_id=1; /// station id of the query server optional string station_id_request=2; /// station id found in the request optional string message=3; /// message found in the request optional int64 local_timestamp=4; /// local timestamp at server, used to detect drifts optional string local_timestamp_string=5; /// on server with time zone, use this optional bool is_primary=6; /// is this a primary or HA node } /// ErrorResponse /// All XYZRequest() messages can either generate a XYZResponse() or an ErrorResponse() /// you need to handle the error case message ErrorResponse{ required int64 original_command=1; /// Command ID of request required int64 error_code=2; /// numeric error code required string error_message=3; /// error string } /// OKResponse /// many messages return an OKResponse indicating success of operation message OKResponse{ required int64 original_command=1; /// command id of request optional string message=2; /// success message } /// CounterItemRequest : Time series history statistics for an item message CounterItemRequest{ required string counter_group=2; /// guid of counter group optional int64 meter=3; /// optional meter, default will retrieve all (same cost) required KeyT key=4; /// key (can specify key.key, key.label, etc too required TimeInterval time_interval=5; /// Time interval for query optional int64 volumes_only=6 [default=0]; /// if '1' ; then only retrieves totals for each meter } /// CounterItemResponse - message CounterItemResponse{ required string counter_group=1; /// guid of CG required KeyT key=2; /// key : filled up with readable,label automatically optional StatsArray totals=3; /// if volumes_only = 1 in request, this contains totals for each metric repeated StatsArray stats=4; /// time series stats - can use to draw charts etc } /// CounterGroupTopperRequest - retrieve toppers for a counter group (top-K) message CounterGroupTopperRequest{ required string counter_group=2; /// guid of CG optional int64 meter=3 [default=0]; /// meter; eg to get Top Hosts By Connections use cg=Hosts meter = 6(connections) optional int64 maxitems=4 [default=100]; /// number of top items to retreive optional TimeInterval time_interval=5; /// time interval , leave blank for latest interval optional Timestamp time_instant=6; /// optional int64 flags=7; optional bool resolve_keys=8 [default=true]; /// retrieve labels as set in the response for each key optional string key_filter=9; /// only get keys with this pattern and SYS:GROUP optional string inverse_key_filter=10; /// only get keys without this pattern NOT } /// CounterGroupTopperResponse message CounterGroupTopperResponse{ required string counter_group=2; /// request cgid required int64 meter=3; /// from request optional int64 sysgrouptotal=4; /// the metric value for "Others.." after Top-K repeated KeyT keys=6; /// topper keys, KeyT.metric contains the top-k value } /// SearchkeysRequest - search for keys message SearchKeysRequest{ required string counter_group=2; optional int64 maxitems=3[default=100]; optional string pattern=4; optional string label=5; repeated string keys=6; optional int64 offset=7[default=0]; optional bool get_totals=8[default=false]; optional bool get_attributes=9[default=false]; } /// SearchKeysResponse message SearchKeysResponse{ required string counter_group=2; repeated KeyT keys=3; optional int64 total_count=4; } /// CounterGroupInfoRequest - retrieve information about enabled counter groups message CounterGroupInfoRequest{ optional string counter_group=2; optional bool get_meter_info=3[default=false]; } /// CounterGroupInfoResponse message CounterGroupInfoResponse{ repeated CounterGroupT group_details=2; } /// QuerySessions - Query flows /// fields filled are treated as AND criteria /// See SessionT for description of common query fields message QuerySessionsRequest { optional string session_group=2[default="{99A78737-4B41-4387-8F31-8077DB917336}"]; optional TimeInterval time_interval=3; optional string key=4; optional KeyT source_ip=5; optional KeyT source_port=6; optional KeyT dest_ip=7; optional KeyT dest_port=8; optional KeyT any_ip=9; /// source or dest match optional KeyT any_port=10; /// source or dest match repeated KeyT ip_pair=11; /// array of 2 ips optional KeyT protocol=12; optional string flowtag=13; /// string flow tagger text optional KeyT nf_routerid=14; optional KeyT nf_ifindex_in=15; optional KeyT nf_ifindex_out=16; optional string subnet_24=17; /// ip /24 subnet matching optional string subnet_16=18; /// ip /16 subnet optional int64 maxitems=19[default=100]; /// maximum number of matching flows to retrieve optional int64 volume_filter=20[default=0]; /// only retrieve flows > this many bytes (a+z) optional bool resolve_keys=21[default=true]; optional string outputpath=22; /// write results to a file (CSV) on trisul-hub (for very large dumps) repeated string idlist=23; /// array of flow ids , usually from SessionTracker response optional KeyT any_nf_ifindex=24; /// composite query for both ifindex } /// QuerySessionsResponse /// a list of matching flows message QuerySessionsResponse { required string session_group=2; repeated SessionT sessions=3; /// matching flows SessionT objects optional string outputpath=4; /// if 'outputpath' set in request, the sessions are here (in CSV format) } /// AggregateSessions - count star /// query flow and then count the numbers /// by default aggregate on ALL fields message AggregateSessionsRequest { optional string session_group=2[default="{99A78737-4B41-4387-8F31-8077DB917336}"]; optional TimeInterval time_interval=3; optional KeyT source_ip=5; optional KeyT source_port=6; optional KeyT dest_ip=7; optional KeyT dest_port=8; optional KeyT any_ip=9; /// source or dest match optional KeyT any_port=10; /// source or dest match repeated KeyT ip_pair=11; /// array of 2 ips optional KeyT protocol=12; optional string flowtag=13; /// string flow tagger text optional KeyT nf_routerid=14; optional KeyT nf_ifindex_in=15; optional KeyT nf_ifindex_out=16; optional string subnet_24=17; /// ip /24 subnet matching optional string subnet_16=18; /// ip /16 subnet optional int64 aggregate_topcount=19[default=100]; /// number of count-star per field repeated string group_by_fields=20; /// list of field names optional KeyT any_nf_ifindex=21; /// matches either gen2 or gen3 } /// AggregateSessionsResponse /// contains counts of each key message AggregateSessionsResponse { message KeyTCount { required KeyT key=1; /// aggregate key required int64 count=2; /// number of matching flows required int64 metric=3; /// aggregated metrics, eg total bytes } message TagGroup { required string group_name=1; repeated KeyTCount tag_metrics=2; } required string session_group=2; optional TimeInterval time_interval=3; repeated KeyTCount source_ip=5; /// top IPs repeated KeyTCount source_port=6; /// top ports repeated KeyTCount dest_ip=7; /// dest IPs repeated KeyTCount dest_port=8; /// dest ports repeated KeyTCount any_ip=9; /// source or dest match repeated KeyTCount any_port=10; /// source or dest match repeated KeyTCount ip_pair=11; /// array of 2 ips repeated KeyTCount protocol=12; /// protocols repeated KeyTCount flowtag=13; /// string flow tagger text repeated KeyTCount nf_routerid=14; repeated KeyTCount nf_ifindex_in=15; repeated KeyTCount nf_ifindex_out=16; repeated KeyTCount subnet_24=17; /// ip /24 subnet matching repeated KeyTCount internal_port=18; /// internal IP ports repeated KeyTCount internal_ip=19; /// internal IPs repeated KeyTCount external_ip=20; /// external IPs repeated TagGroup tag_group=21; /// tag groups } /// UpdatekeysRequest /// Response = OKResponse or ErrorResponse message UpdateKeyRequest{ required string counter_group=2; repeated KeyT keys=4; /// key : if you set both key and label, the DB label will be updated optional bool remove_all_attributes=5; /// remove all key attributes from keys repeated string remove_attributes=6; /// remove these attributes only from keys } /// SessionTrackerRequest - query session trackers /// session trackers are top-k streaming algorithm for network flows /// They are Top Sessions fulfilling a particular preset criterion message SessionTrackerRequest { optional string session_group=2[default="{99A78737-4B41-4387-8F31-8077DB917336}"]; optional int64 tracker_id=3; /// session tracker id optional int64 maxitems=4 [default=100]; required TimeInterval time_interval=5; optional bool resolve_keys=6 [default=true]; optional string tracker_name=7; /// query by tracker name for auto generated trackers } /// SessionTrackerResponse - results of tracker /// returns a list of SessionT for the matching sessions. /// Note: the returned list of SessionT only contains keys (in key format) and the /// tracker_statval reprsenting the tracker metric. You need to send further QuerySession /// request with the session_key to retrive the fullflow message SessionTrackerResponse{ required string session_group=2; repeated SessionT sessions=3; /// contains session_key and tracker_statval optional int64 tracker_id=4; } /// QueryAlertsRequest - query alerts in system, can group_by (aggregate) any one field /// multiple query fields are treated as AND message QueryAlertsRequest { required string alert_group=2; optional TimeInterval time_interval=3; optional int64 maxitems=5 [default=100]; optional KeyT source_ip=6; optional KeyT source_port=7; optional KeyT destination_ip=8; optional KeyT destination_port=9; optional KeyT sigid=10; optional KeyT classification=11; optional KeyT priority=12; optional string aux_message1=13; /// matches dispatchmessage1 in AlertT optional string aux_message2=14; /// matches dispatchmessage2 in AlertT optional string group_by_fieldname=15; /// can group by any field - group by 'sigid' will group results by sigid repeated string idlist=16; /// list of alert ids optional bool resolve_keys=17[default=true]; optional KeyT any_ip=18; /// search by any_ip (source_dest) optional KeyT any_port=19; /// search by any_port (source_dest) repeated KeyT ip_pair=20; /// array of 2 ips optional string message_regex=21; /// searech via regex of the dispatch message optional bool approx_count_only=22[default=false]; /// approx count per alert group } /// QueryAlertsResponse - response /// if you used group_by_fieldname then AlertT.occurrances would contain the count message QueryAlertsResponse { required string alert_group=2; repeated AlertT alerts=3; /// array of matching alerts optional int64 approx_count=4; /// approx count } /// QueryResourcesRequest - resource queries /// DNS, HTTP, message QueryResourcesRequest { required string resource_group=2; optional TimeInterval time_interval=3; optional int64 maxitems=4 [default=100]; optional KeyT source_ip=5; optional KeyT source_port=6; optional KeyT destination_ip=7; optional KeyT destination_port=8; optional string uri_pattern=9; /// DEPRECATED: use regex_uri for everything optional string userlabel_pattern=10; /// DEPRECATED: uri_pattern will search both URI and USERLABEL repeated string regex_uri=12; /// cant be combined with others repeated string idlist=13; /// resource ID list optional bool resolve_keys=14 [default=true]; optional KeyT any_port=15; optional KeyT any_ip=16; repeated KeyT ip_pair=17; // array of 2 ips optional bool approx_count_only=18[default=false]; // count-star only over time interval repeated string exclude_iplist=19; /// Exclude these IPs, cant be used with other IP fields optional bool invert_regex=20[default=false]; /// Invert the regex RE2 cant reject } /// QueryResourceResponse message QueryResourcesResponse { required string resource_group=2; repeated ResourceT resources=3; optional int64 approx_count=4; } /// AggregatedResourceRequest - resource queries /// DNS, HTTP, message AggregateResourcesRequest { required QueryResourcesRequest query=1; optional int64 aggregate_topcount=2[default=100]; /// number of count-star per field } /// AggregateResourcesResponse /// various counters message AggregateResourcesResponse { message KeyTCount { required KeyT key=1; required int64 count=2; } repeated KeyTCount source_ip=5; repeated KeyTCount source_port=6; repeated KeyTCount destination_ip=7; repeated KeyTCount destination_port=8; repeated KeyTCount uri=9; /// group by URI repeated KeyTCount userlabel=10; /// group by Pattern } /// KeySpaceRequest - search hits in Key Space /// for example you can search the key space 10.0.0.0 to 11.0.0.0 to get all IP /// seen in that range message KeySpaceRequest { required string counter_group=2; required TimeInterval time_interval=3; optional int64 maxitems=4 [default=100]; message KeySpace { required KeyT from_key=1; /// from key representing start of keyspace required KeyT to_key=2; /// end of key space } repeated KeySpace spaces=5; /// space1 OR space2 OR .. optional bool resolve_keys=6[default=true]; /// resolve names optional bool totals_only=7[default=false]; /// if set, only totals returned } /// KeySpaceResponse message KeySpaceResponse { optional string counter_group=2; /// counter group repeated KeyT hits=3; /// array of keys in the requested space optional int64 total_hits=4; /// if totals_only() this contains total(hits) } /// TopperTrendRequest - raw top-K at each topper snapshot interval /// can use this to see "Top apps over 1 Week" message TopperTrendRequest { required string counter_group=2; optional int64 meter=3 [default=0]; optional int64 maxitems=4 [default=100]; optional TimeInterval time_interval=5; optional string key_filter=6; /// only get keys with this pattern and SYS:GROUP optional string inverse_key_filter=7; /// only get keys without this pattern NOT } /// TopperTrendResponse message TopperTrendResponse { required string counter_group=2; required int64 meter=3; repeated KeyStats keytrends=4; /// timeseries - ts, (array of key stats) for each snapshot interval } /// Subscribe - add a subcription to the Real Time channel message SubscribeCtl { // from TrisulAPI enum StabberType { ST_COUNTER_ITEM=0; ST_ALERT=1; ST_FLOW=2; ST_TOPPER=3; } enum CtlType { CT_SUBSCRIBE=0; CT_UNSUBSCRIBE=1; } required string context_name=1; required CtlType ctl=2; required StabberType type=3; optional string guid=4; optional string key=5; optional int64 meterid=6; } /// FTS /// query to return docs, docids, and flows based on keyword search /// message QueryFTSRequest { required TimeInterval time_interval=2; required string fts_group=3; required string keywords=4; optional int64 maxitems=5[default=100]; optional bool approx_count_only=6[default=false]; } message QueryFTSResponse { required string fts_group=2; repeated DocumentT documents=3; optional int64 approx_count=4; /// approx count } /// Timeslices - retrieves the backend timeslice details /// /// get the METERS METASLICE info /// .. response = TimeSlicesResponse message TimeSlicesRequest { optional bool get_disk_usage=1[default=false]; /// get disk usage per SLICE optional bool get_all_engines=2[default=false]; /// by default (set to false) we only get instance-0 /// because are interested in iterating over /// the time interval rather then each slice /// inside the time interval. set this to true /// if you want each slice to be reported optional bool get_total_window=3[default=false]; /// just get the total time window in the /// trisul database } /// .. response = TimeSlicesResponse /// get the PCAP METASLICE based info message PcapSlicesRequest { required string context_name=1; optional bool get_total_window=2[default=false]; } message TimeSlicesResponse { message SliceT { required TimeInterval time_interval=1; optional string name=2; optional string status=3; optional int64 disk_size=4; optional string path=5; optional bool available=6; }; repeated SliceT slices=1; optional TimeInterval total_window=2; optional string context_name=3; } /// DeleteAlerts /// - very limited exception to Trisul rule of not having delete options message DeleteAlertsRequest { required string alert_group=2; required TimeInterval time_interval=3; optional KeyT source_ip=6; optional KeyT source_port=7; optional KeyT destination_ip=8; optional KeyT destination_port=9; optional KeyT sigid=10; optional KeyT classification=11; optional KeyT priority=12; optional KeyT any_ip=18; optional KeyT any_port=19; optional string message_regex=21; /// delete using regex } /// MetricsSummaryRequest - used to retrieve DB stats message MetricsSummaryRequest{ optional TimeInterval time_interval=1; required string metric_name=2; optional bool totals_only=3[default=true]; } /// MetricsSummaryResponse message MetricsSummaryResponse { required string metric_name=2; repeated StatsTuple vals=3; } /// LogRequest - get log file from a domain node message LogRequest { required string context_name=1; required string log_type=2; optional string regex_filter=4; optional int64 maxlines=5[default=1000]; optional string continue_logfilename=6; optional int64 continue_seekpos=7; optional bool latest_run_only=8[default=false]; } message LogResponse { required string context_name=1; optional string logfilename=6; optional int64 seekpos=7; repeated string log_lines=8; /// compressed gz } /// messages to routerX backend message DomainRequest { required DomainOperation cmd=1; optional string station_id=2; optional string params=3; optional DomainNodeType nodetype=4; } message DomainResponse { message Node { required string id=1; required DomainNodeType nodetype=2; optional string station_id=3; optional string extra_info=4; optional Timestamp register_time=5; optional Timestamp heartbeat_time=6; optional bool is_primary=7[default=true]; /// primary or backup HA mode } required DomainOperation cmd=1; repeated Node nodes=2; optional string req_params=3; optional string params=4; optional bool need_reconnect=5[default=false]; } /// messages to HAControl message HAControlRequest { enum HAOperation { HA_TEST_REACHABILITY=0; /// test if primary or backup is reachable (test the other side) HA_SWITCH_BACKUP=1; /// switch all nodes to backup HA HA_SWITCH_PRIMARY=2; /// switch to primary HA } required HAOperation cmd=1; optional string station_id=2; optional string params=3; } message HAControlResponse { optional bool control_success=1; optional string station_id=2; optional string status_message=3; optional bool primary_reachable=4; optional bool backup_reachable=5; } /// tool info /// general purpose probe information gathering tool /// use cases : get BGP route receiver peer status message ToolInfoRequest { optional string context_name=1; optional string tool_name=2; repeated string tool_info_requested=3; /// array of information elements requested } message ToolInfoResponse { repeated NameValue tool_info=1; /// info elements Name=Value } message NodeConfigRequest { optional string message=1; message IntelFeed { required string guid=1; /// identifying feed group (eg Geo, Badfellas) optional string name=2; /// feed name (eg badfellas) optional string download_rules=3; /// xml file with feed update instructions repeated string uri=4; /// individual files in config//.. for FileRequest download repeated string usernodes=5; /// user requesting the feed, the probe0, probe1, etc optional int64 sub_feed_id=6[default=-1];/// sub feed ID, when a feed consists of several sub-feeds optional bool restore_mode=7[default=false]; /// restore mode will backup and restore, used with add,remove } optional IntelFeed add_feed=2; optional IntelFeed process_new_feed=3; optional bool get_all_nodes=4[default=true]; repeated NameValue query_config=5; optional IntelFeed remove_feed=6; } message NodeConfigResponse { message Node { required string id=1; required DomainNodeType nodetype=2; required string description=3; required string public_key=4; } repeated Node domains=1; repeated Node hubs=2; repeated Node probes=3; repeated string feeds=4; repeated NameValue config_values=5; } /// ContextRequest - Context methods /// response Ok or Error, follow up with ContextInfo to print details /// message ContextCreateRequest { required string context_name=1; optional string clone_from=2; } /// ContextInfo : one or all contexts /// use is_init to prime with config message ContextInfoRequest { optional string context_name=1; /// if not set all context get in optional bool get_size_on_disk=2[default=false]; /// get size on disk (expensive) optional string tool_name=3; /// get status of this tool, freeform bgp, suricata, snort, udpsink etc.. } message ContextInfoResponse { message Item { required string context_name=1; required bool is_initialized=2; required bool is_running=3; /// trisul processes runs optional int64 size_on_disk=4; optional TimeInterval time_interval=5; optional bool is_clean=6; optional string extrainfo=7; /// freeform text output when tool is specified , can be long string repeated TimeInterval run_history=8; optional string profile=9; optional string runmode=10; optional string node_version=11; } repeated Item items=1; } /// ContextDelete : initialize /// reset data only .. message ContextDeleteRequest { required string context_name=1; /// if not set all context get in optional bool reset_data=2; /// reset data dont delete everything } /// ContextStart : run /// run data only .. message ContextStartRequest { required string context_name=1; /// if not set all context get in optional string mode=2; /// same as trisul cmdline run mode optional bool background=3[default=true]; optional string pcap_path=4; optional string run_tool=5; /// snort, suricata supported.. optional string tool_ids_config=6; optional string tool_av_config=7; optional string cmd_in=8; /// maps to trisul -in optional string cmd_out=9; /// maps to trisul -out optional string cmd_args=10; /// maps to trisul -args } /// ContextSttop : kill the context processes message ContextStopRequest { required string context_name=1; /// if not set all context get in optional string run_tool=5; /// snort, suricata , trp, flushd supported.. } /// ContextConfigRequest - start stop status /// OK or ERROR response /// Status = OK if running with PID etc in message text message ContextConfigRequest { required string context_name=1; optional string profile=2; optional string params=3; optional bytes push_config_blob=4; /// push this .. repeated NameValue query_config=5; /// query, leave the .value field blank repeated NameValue set_config_values=6; /// push this .. (name=value;name=value ..) } message ContextConfigResponse { message Layer { required int64 layer=1; required string probe_id=2; optional string probe_description=3; } required string context_name=1; optional string profile=2; optional string params=3; /// what kind of config you want optional bytes pull_config_blob=4; /// config optional bytes config_blob=5; /// compress tar.gz .. repeated string endpoints_flush=6; repeated string endpoints_query=7; repeated string endpoints_pub=8; repeated NameValue config_values=10; /// query, leave the .value field blank repeated Layer layers=11; } /// PcapRequest - retrieve a PCAP /// Sent directly to each probe rather than to the DB query HUB /// /// the flow is PCAP Request for a file -> put a file on the probe > return a token /// > use that token in FileRequest to download the file from the probe /// /// see app notes and examples /// /// NOTE - only one of the various filters are supported /// sending > 1 will result in error /// /// Modes /// 1. nothing set => PCAP file in contents /// 2. save_file_prefix set => file download token /// 3. merge_pcap_files => file download token /// /// message PcapRequest { required string context_name=1; // context optional int64 max_bytes=2[default=100000000]; // max return PCAP size default=100MB , can increase to 0.75 Filesystem freespace optional CompressionType compress_type=3[default=UNCOMPRESSED]; optional TimeInterval time_interval=4; // not needed for merge option optional string save_file_prefix=5; optional string filter_expression=6; /// PCAP filter expression in Trisul Filter format repeated string merge_pcap_files=7; /// list of PCAP files on probe that you need to merge optional bool delete_after_merge=8[default=true]; optional PcapFormat format=9[default=LIBPCAP]; } /// Pcap Response - for small files (<1MB) contents directly contain the PCAP /// for larger files, save_file contains a download token for use by FileRequest message PcapResponse { required string context_name=1; optional PcapFormat format=2[default=LIBPCAP]; optional CompressionType compress_type=3[default=UNCOMPRESSED]; optional TimeInterval time_interval=4; optional int64 num_bytes=5; optional string sha1=6; optional bytes contents=7; optional string save_file=8; //use FileRequest framework to download } /// GrepRequest - reconstruct and search for patterns in saved packets message GrepRequest { required string context_name=1; required TimeInterval time_interval=2; optional int64 maxitems=3 [default=100]; optional int64 flowcutoff_bytes=4; optional string pattern_hex=5; /// hex patttern optional string pattern_text=6; /// plain text optional string pattern_file=7; /// a file - must be available at probe repeated string md5list=8; /// a list of MD5 matching the content optional bool resolve_keys=9 [default=true]; } /// GrepResponse message GrepResponse { required string context_name=1; repeated SessionT sessions=2; /// sessionT with keys containing the content repeated string hints=3; /// some surrounding context for the match optional string probe_id=4; } /// ProbeStatsRequest - DOMAIN /// retrieve statistics about probe cpu, mem, etc message ProbeStatsRequest{ required string context_name=1; optional string param=2; } /// ProbeStatsResponse message ProbeStatsResponse { required string context_name=1; required string instance_name=2; required int64 connections=3; required int64 uptime_seconds=4; required double cpu_usage_percent_trisul=5; required double cpu_usage_percent_total=6; required double mem_usage_trisul=7; required double mem_usage_total=8; required double mem_total=9; required double drop_percent_cap=10; required double drop_percent_trisul=11; optional int64 proc_bytes=12; optional int64 proc_packets=13; optional string offline_pcap_file=14; optional bool is_running=15; } /// AsyncResponse - a token represnting a future response /// you will get an AsyncResponse for TRP Request if you set the run_async=true at the message level message AsyncResponse { required int64 token=1; /// use this token in AsyncRequest polling until you get the original Response you expected optional string response_message=3; optional Message response=4; } /// AsyncRequest - Asynchrononous query framework /// response taken from original , the token message AsyncRequest { required int64 token=1; // token from AsyncResponse optional string request_message =2; // basically extra text for logging } /// FileRequest - used to download files from Trisul domain nodes like probes message FileRequest { required string uri=1; /// uri of resource you want to download , example PcapResponse.save_file required int64 position=2; /// seek position in that file optional string params=3; /// local meaning sentback n response optional string context_name=4; /// context name optional bool delete_on_eof=5[default=false]; } /// FileResponse /// one chunk at at time, Trisul has slightly inefficient File Transfer /// for very large files, since most files are data feeds < 100MB fine for now message FileResponse { required string uri=1; /// requested URI required bool eof=2; /// end of all chunks optional int64 position=3; /// current position optional bytes content=4; /// file chunk content optional string request_params =5; optional string context_name=6; } /// GraphRequest /// given a subject node, retrive a graph for a given time window message GraphRequest { required TimeInterval time_interval=1; /// time window required string subject_group=2; /// guid of subject, eg counter or alert guid required KeyT subject_key=3; /// key (can specify key.key, key.label, etc too optional string vertex_group=4; /// optional vertex group (only get vertices in this group) } message GraphResponse { required string subject_group=1; /// from request required KeyT subject_key=2; /// from request repeated EdgeGraphT graphs=3; /// graphs - an EdgeGraphT message /// repeated because returned in time chunks } /// RunToolRequest : run set of named tools message RunToolRequest { required string context_name=1; /// if not set all context get in enum NodeTool { PING=1; DF=2; GEOQUERY=3; TOP=4; BGPQUERY=5; } required NodeTool tool=2; /// which of these tools you want to run on node optional string tool_input=3; /// input optional string tool_input_file_data=4;/// data that goes into a tool input file } /// RunToolResponse : ouput message RunToolResponse { required string context_name=1; /// if not set all context get in optional string tool_output=2; /// cmd lines }