Wednesday, May 6, 2015

FortiGate Extractors for Graylog 1.0

Graylog is a nice opensource alternative to Splunk and other SIEM tools. I've been using it for several years, and continue to make tweaks to improve its usefulness in my environment. I'm excited now that it is on version 1.0 (and was renamed Graylog instead of Graylog2), and is a lot more stable.

One of the tweaks I made a while back on a previous version was to create create a DRL extractor for FortiGate (a firewall made by FortiNet). I've now updated this extractor so that you can import it using the new JSON format directly into the web interface (instead of having to create the DRL file, etc).

To apply the extractors on Graylog, go to your FortiGate Input, and Import Extractors. The details on how to do that can be found on Graylog's site here.

Here's the JSON script for the extractors:

{
  "extractors": [
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sdevname=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "source",
      "title": "FGTsource"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\saction=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "action",
      "title": "FGTaction"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sapp=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "app",
      "title": "FGTapp"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sappact=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "appact",
      "title": "FGTappact"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sappcat=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "appcat",
      "title": "FGTappcat"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sapplist=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "applist",
      "title": "FGTapplist"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sattack=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "attack",
      "title": "FGTattack"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sdevid=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "devid",
      "title": "FGTdevid"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sdir=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "dir",
      "title": "FGTdir"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sdstcountry=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "dstcountry",
      "title": "FGTdstcountry"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sdstintf=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "dstintf",
      "title": "FGTdstintf"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sdstip=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "dstip",
      "title": "FGTdstip"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sdstport=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "dstport",
      "title": "FGTdstport"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sdtype=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "dtype",
      "title": "FGTdtype"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sduration=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "duration",
      "title": "FGTduration"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\serror_reason=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "error_reason",
      "title": "FGTerror_reason"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\seventtype=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "eventtype",
      "title": "FGTeventtype"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sfile=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "file",
      "title": "FGTfile"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sgroup=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "group",
      "title": "FGTgroup"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\shostname=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "hostname",
      "title": "FGThostname"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sidentidx=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "identidx",
      "title": "FGTidentidx"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sinit=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "init",
      "title": "FGTinit"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\slocip=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "locip",
      "title": "FGTlocip"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\slocport=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "locport",
      "title": "FGTlocport"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\slogid=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "logid",
      "title": "FGTlogid"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\smode=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "mode",
      "title": "FGTmode"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\smsg=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "msg",
      "title": "FGTmsg"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\soutintf=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "outintf",
      "title": "FGToutintf"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\speer_notif=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "peer_notif",
      "title": "FGTpeer_notif"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\spolicyid=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "policyid",
      "title": "FGTpolicyid"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sprofile=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "profile",
      "title": "FGTprofile"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sprofiletype=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "profiletype",
      "title": "FGTprofiletype"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sproto=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "proto",
      "title": "FGTproto"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\squarskip=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "quarskip",
      "title": "FGTquarskip"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\srcvdbyte=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "rcvdbyte",
      "title": "FGTrcvdbyte"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\srcvdpkt=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "rcvdpkt",
      "title": "FGTrcvdpkt"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sref=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "ref",
      "title": "FGTref"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sremip=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "remip",
      "title": "FGTremip"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sremport=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "remport",
      "title": "FGTremport"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sresult=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "result",
      "title": "FGTresult"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\srole=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "role",
      "title": "FGTrole"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\ssentbyte=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "sentbyte",
      "title": "FGTsentbyte"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\ssentpkt=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "sentpkt",
      "title": "FGTsentpkt"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sservice=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "service",
      "title": "FGTservice"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sservice=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "service",
      "title": "FGTservice"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\ssrccountry=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "srccountry",
      "title": "FGTsrccountry"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\ssrcintf=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "srcintf",
      "title": "FGTsrcintf"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\ssrcip=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "srcip",
      "title": "FGTsrcip"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\ssrcport=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "srcport",
      "title": "FGTsrcport"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sstage=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "stage",
      "title": "FGTstage"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sstatus=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "status",
      "title": "FGTstatus"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sstatus=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "status",
      "title": "FGTstatus"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\ssubtype=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "subtype",
      "title": "FGTsubtype"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\stransport=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "transport",
      "title": "FGTtransport"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\stype=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "type",
      "title": "FGTtype"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\strandisp=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "trandisp",
      "title": "FGTtrandisp"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\stransip=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "transip",
      "title": "FGTtransip"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\suser=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "user",
      "title": "FGTuser"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sutmaction=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "utmaction",
      "title": "FGTutmaction"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sutmevent=(\\S+)\\s"
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "utmevent",
      "title": "FGTutmevent"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\svd=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "vd",
      "title": "FGTvd"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": ".+\\svirus=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "virus",
      "title": "FGTvirus"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\svpntunnel=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "vpntunnel",
      "title": "FGTvpntunnel"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sxauthgroup=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "xauthgroup",
      "title": "FGTxauthgroup"
    },
    {
      "condition_type": "none",
      "condition_value": "",
      "converters": [],
      "cursor_strategy": "copy",
      "extractor_config": {
        "regex_value": "^.+\\sxauthuser=\\\"(.+?)\\\""
      },
      "extractor_type": "regex",
      "order": 0,
      "source_field": "message",
      "target_field": "xauthuser",
      "title": "FGTxauthuser"
    }
  ],
  "version": "1.0.0"
}

------
Dustin Shaw
VCP

3 comments:

  1. What input method did you use for your fortinet devices?

    ReplyDelete
    Replies
    1. I simply created a new Syslog input on a different port for my FortiNet devices. Then I configured the syslog server on the FortiGate (instructions here: http://docs-legacy.fortinet.com/fos50hlp/50/index.html#page/FortiOS%205.0%20Help/monitoring.102.13.html ) to send the data to that port.

      Delete