使用 JQ 解析 aws cli 输出安全组答案

【问题标题】：Parse aws cli output security groups with JQ使用 JQ 解析 aws cli 输出安全组
【发布时间】：2014-10-24 07:27:00
【问题描述】：

我开始疯狂地使用“JQ”解析来自 AWS CLI 的 json 输出。

也许我还不明白如何正确使用 JQ，但我喜欢解析 2 个（或更多）安全组，每个安全组都为每个规则格式化一个字符串，包括入站和出站

AWS CLI 命令是这样的：

aws ec2 describe-security-groups --group-ids sg-0000001 sg-0000002

输出的 json 是关于这个的（我随机编辑了一些隐私信息）：

{
    "SecurityGroups": [
        {
            "IpPermissionsEgress": [
                {
                    "IpProtocol": "-1",
                    "IpRanges": [
                        {
                            "CidrIp": "0.0.0.0/0"
                        }
                    ],
                    "UserIdGroupPairs": []
                }
            ],
            "Description": "server-db",
            "Tags": [
                {
                    "Value": "server-db",
                    "Key": "Client"
                },
                {
                    "Value": "server-db",
                    "Key": "Name"
                }
            ],
            "IpPermissions": [
                {
                    "ToPort": 3389,
                    "IpProtocol": "tcp",
                    "IpRanges": [
                        {
                            "CidrIp": "10.12.0.0/16"
                        },
                        {
                            "CidrIp": "192.168.10.10/32"
                        }
                    ],
                    "UserIdGroupPairs": [],
                    "FromPort": 3389
                },
                {
                    "ToPort": 5666,
                    "IpProtocol": "tcp",
                    "IpRanges": [
                        {
                            "CidrIp": "192.168.10.10/32"
                        }
                    ],
                    "UserIdGroupPairs": [],
                    "FromPort": 5666
                },
                {
                    "IpProtocol": "-1",
                    "IpRanges": [],
                    "UserIdGroupPairs": [
                        {
                            "UserId": "121211212121",
                            "GroupId": "sg-00000001"
                        }
                    ]
                },
                {
                    "IpProtocol": "-1",
                    "IpRanges": [],
                    "UserIdGroupPairs": [
                        {
                            "UserId": "121211212121",
                            "GroupId": "sg-000000001"
                        }
                    ]
                },
                {
                    "ToPort": -1,
                    "IpProtocol": "icmp",
                    "IpRanges": [
                        {
                            "CidrIp": "10.12.0.0/16"
                        },
                        {
                            "CidrIp": "192.168.10.10/32"
                        }
                    ],
                    "UserIdGroupPairs": [],
                    "FromPort": -1
                }
            ],
            "GroupName": "server-db",
            "VpcId": "vpc-0000001",
            "OwnerId": "121211212121",
            "GroupId": "sg-000000001"
        },
        {
            "IpPermissionsEgress": [
                {
                    "IpProtocol": "-1",
                    "IpRanges": [
                        {
                            "CidrIp": "0.0.0.0/0"
                        }
                    ],
                    "UserIdGroupPairs": []
                }
            ],
            "Description": "server-as",
            "Tags": [
                {
                    "Value": "server-as",
                    "Key": "Name"
                },
                {
                    "Value": "server",
                    "Key": "Client"
                }
            ],
            "IpPermissions": [
                {
                    "IpProtocol": "-1",
                    "IpRanges": [],
                    "UserIdGroupPairs": [
                        {
                            "UserId": "121211212121",
                            "GroupId": "sg-00000001"
                        }
                    ]
                },
                {
                    "ToPort": 22,
                    "IpProtocol": "tcp",
                    "IpRanges": [
                        {
                            "CidrIp": "10.12.0.0/16"
                        },
                        {
                            "CidrIp": "192.168.10.10/32"
                        }
                    ],
                    "UserIdGroupPairs": [],
                    "FromPort": 22
                },
                {
                    "ToPort": 443,
                    "IpProtocol": "tcp",
                    "IpRanges": [
                        {
                            "CidrIp": "10.12.0.0/16"
                        },
                        {
                            "CidrIp": "192.168.60.10/32"
                        },
                        {
                            "CidrIp": "192.168.160.10/32"
                        },
                        {
                            "CidrIp": "192.168.130.10/32"
                        },
                        {
                            "CidrIp": "192.168.130.50/32"
                        },
                        {
                            "CidrIp": "192.168.130.150/32"
                        },
                        {
                            "CidrIp": "192.168.10.10/32"
                        },
                        {
                            "CidrIp": "192.168.80.150/32"
                        },
                        {
                            "CidrIp": "192.168.80.152/32"
                        },
                        {
                            "CidrIp": "192.168.80.155/32"
                        },
                        {
                            "CidrIp": "192.168.80.158/32"
                        }
                    ],
                    "UserIdGroupPairs": [],
                    "FromPort": 443
                },
                {
                    "IpProtocol": "-1",
                    "IpRanges": [],
                    "UserIdGroupPairs": [
                        {
                            "UserId": "121211212121",
                            "GroupId": "sg-00000002"
                        }
                    ]
                },
                {
                    "ToPort": -1,
                    "IpProtocol": "icmp",
                    "IpRanges": [
                        {
                            "CidrIp": "10.12.0.0/16"
                        },
                        {
                            "CidrIp": "192.168.10.10/32"
                        }
                    ],
                    "UserIdGroupPairs": [],
                    "FromPort": -1
                }
            ],
            "GroupName": "server-as",
            "VpcId": "vpc-00000001",
            "OwnerId": "121211212121",
            "GroupId": "sg-00000001"
        }
    ]
}

id 喜欢的格式可能是：

SecurityGroupId - GroupName - InBound/OutBound - IpProtocol - Port - SourceRanges/DestinationRanges

有人可以帮助我吗？谢谢

【问题讨论】：

我可以构建的唯一过滤器是 .["SecurityGroups"] | .[] | .GroupName +" "+ .GroupId

标签： json amazon-web-services jq

【解决方案1】：

这是一个jq解决方案

def ports:
   if .IpProtocol == "-1"
   then {FromPort:"ALLPORTS", ToPort:"ALLPORTS", IpProtocol:"ALLPROTO"}
   else {FromPort, ToPort, IpProtocol}
   end
;

def tabella($g; $p; $dir; $ip):
    [ $g.GroupId, $g.GroupName, $g.Description, $g.VpcId, $dir, $ip,
      $p.FromPort, $p.ToPort, $p.IpProtocol ]
;

def creatab:
      .SecurityGroups[]
    | { GroupId, GroupName, Description, VpcId } as $g
    | (
          .IpPermissions[]
        | ports as $p
        | ( .IpRanges[]         | tabella($g; $p; "INBOUND"; .CidrIp) ),
          ( .UserIdGroupPairs[] | tabella($g; $p; "INBOUND"; .GroupId) )
      ),
      (
          .IpPermissionsEgress[]
        | ports as $p
        | ( .IpRanges[]         | tabella($g; $p; "OUTBOUND"; .CidrIp) ),
          ( .UserIdGroupPairs[] | tabella($g; $p; "OUTBOUND"; .GroupId) )
      )
;

  creatab
| map(tostring)
| join("|")

当运行时

jq -M -r -f filter.jq data.json

它将产生与Marco's perl solution相同的输出

sg-000000001|server-db|server-db|vpc-0000001|INBOUND|10.12.0.0/16|3389|3389|tcp
sg-000000001|server-db|server-db|vpc-0000001|INBOUND|192.168.10.10/32|3389|3389|tcp
sg-000000001|server-db|server-db|vpc-0000001|INBOUND|192.168.10.10/32|5666|5666|tcp
sg-000000001|server-db|server-db|vpc-0000001|INBOUND|sg-00000001|ALLPORTS|ALLPORTS|ALLPROTO
sg-000000001|server-db|server-db|vpc-0000001|INBOUND|sg-000000001|ALLPORTS|ALLPORTS|ALLPROTO
sg-000000001|server-db|server-db|vpc-0000001|INBOUND|10.12.0.0/16|-1|-1|icmp
sg-000000001|server-db|server-db|vpc-0000001|INBOUND|192.168.10.10/32|-1|-1|icmp
sg-000000001|server-db|server-db|vpc-0000001|OUTBOUND|0.0.0.0/0|ALLPORTS|ALLPORTS|ALLPROTO
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|sg-00000001|ALLPORTS|ALLPORTS|ALLPROTO
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|10.12.0.0/16|22|22|tcp
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|192.168.10.10/32|22|22|tcp
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|10.12.0.0/16|443|443|tcp
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|192.168.60.10/32|443|443|tcp
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|192.168.160.10/32|443|443|tcp
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|192.168.130.10/32|443|443|tcp
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|192.168.130.50/32|443|443|tcp
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|192.168.130.150/32|443|443|tcp
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|192.168.10.10/32|443|443|tcp
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|192.168.80.150/32|443|443|tcp
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|192.168.80.152/32|443|443|tcp
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|192.168.80.155/32|443|443|tcp
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|192.168.80.158/32|443|443|tcp
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|sg-00000002|ALLPORTS|ALLPORTS|ALLPROTO
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|10.12.0.0/16|-1|-1|icmp
sg-00000001|server-as|server-as|vpc-00000001|INBOUND|192.168.10.10/32|-1|-1|icmp
sg-00000001|server-as|server-as|vpc-00000001|OUTBOUND|0.0.0.0/0|ALLPORTS|ALLPORTS|ALLPROTO

【讨论】：

【解决方案2】：

用 PERL 解决：

use JSON qw( decode_json );

sub creatab{
my $json = $dump; #dump from the AWS CLI
my $decoded = decode_json($json);

my @secgrp = @{ $decoded->{'SecurityGroups'} };
foreach my $f ( @secgrp ) {
 $description=$f->{"Description"};
 $groupname=$f->{"GroupName"};
 $vpcid=$f->{"VpcId"}; if ($vpcid eq "") {$vpcid = "EC2"}
 $groupid=$f->{"GroupId"};

#------INBOUND RULES------------------------
 my @ipperm = @{ $f->{'IpPermissions'} };
 foreach my $g ( @ipperm ) {
  $toport=$g->{'ToPort'};
  $fromport=$g->{'FromPort'};
  $proto=$g->{'IpProtocol'};

  my @cidr = @{ $g->{'IpRanges'} };
  foreach my $h ( @cidr ) {
   $cidr=$h->{'CidrIp'};
    if ($proto==-1) {$fromport="ALLPORTS"; $toport="ALLPORTS"; $proto="ALLPROTO";}
    $tabella .= "$groupid|$groupname|$description|$vpcid|INBOUND|$cidr|$fromport|$toport|$proto\n";
  }

  my @useridgrouppairs = @{ $g->{'UserIdGroupPairs'} };
  foreach my $h ( @useridgrouppairs ) {
   $useridgrouppairs=$h->{'GroupId'};
    if ($proto==-1) {$fromport="ALLPORTS"; $toport="ALLPORTS"; $proto="ALLPROTO";}
    $tabella .= "$groupid|$groupname|$description|$vpcid|INBOUND|$useridgrouppairs|$fromport|$toport|$proto\n";
  }
 }
#-------------------------------------------
#-------------OUTBOUND RULES----------------
 my @ipperm = @{ $f->{'IpPermissionsEgress'} };
 foreach my $g ( @ipperm ) {
  $toport=$g->{'ToPort'};
  $fromport=$g->{'FromPort'};
  $proto=$g->{'IpProtocol'};

  my @cidr = @{ $g->{'IpRanges'} };
  foreach my $h ( @cidr ) {
   $cidr=$h->{'CidrIp'};
    if ($proto==-1) {$fromport="ALLPORTS"; $toport="ALLPORTS"; $proto="ALLPROTO";}
    $tabella .= "$groupid|$groupname|$description|$vpcid|OUTBOUND|$cidr|$fromport|$toport|$proto\n";
  }

  my @useridgrouppairs = @{ $g->{'UserIdGroupPairs'} };
  foreach my $h ( @useridgrouppairs ) {
   $useridgrouppairs=$h->{'GroupId'};
    if ($proto==-1) {$fromport="ALLPORTS"; $toport="ALLPORTS"; $proto="ALLPROTO";}
    $tabella .= "$groupid|$groupname|$description|$vpcid|OUTBOUND|$useridgrouppairs|$fromport|$toport|$proto\n";
  }
 }
#------------------------------------------
}
return $tabella;
}

【讨论】：