Mailing List Archive

aggregation tool that allows a bit of fuzz to aggregating ?
Hello All , google foo isn't pulling up anything but drivel about ip
aggregation in cisco and describing the word 'fuzz' .

I am looking for a tool such as 'aggregate' , this one is written in
c . Which has been a very good tool to me .

I use this tool to aggregate ip addresses snagged out of various logs to
insert into iptables filtering . Again the afore mentioned tool has work well .

But now I am seeing a new trick fro some entities that are transmitting
from every other ipv4 address such as (*) below . And the trust (& crusty)
ol'tool just doesn't allow for a bitt of fuzz in its aggregation filter .

Hoping someone knows of such a tool and or may have patched the
aggregate tool to accopmlish such a task .

(*)
...
63.81.88.116/32
63.81.88.118/32
63.81.88.120/32
63.81.88.122/32
63.81.88.124/32
63.81.88.126/32
...

Tia , JimL

--
+---------------------------------------------------------------------+
| James W. Laferriere | System Techniques | Give me VMS |
| Network & System Engineer | 3237 Holden Road | Give me Linux |
| jiml@system-techniques.com | Fairbanks, AK. 99709 | only on AXP |
+---------------------------------------------------------------------+
Re: aggregation tool that allows a bit of fuzz to aggregating ? [ In reply to ]
On Sun, 13 Jun 2021 14:47:01 -0800, "babydr DBA James W. Laferriere" said:

> But now I am seeing a new trick fro some entities that are transmitting

> from every other ipv4 address such as (*) below . And the trust (& crusty)
> ol'tool just doesn't allow for a bitt of fuzz in its aggregation filter .
>
> Hoping someone knows of such a tool and or may have patched the
> aggregate tool to accopmlish such a task .
>
> (*)
> ...
> 63.81.88.116/32
> 63.81.88.118/32
> 63.81.88.120/32
> 63.81.88.122/32
> 63.81.88.124/32
> 63.81.88.126/32

Not exactly a fix, but it may relieve the pain until you get one:

cat inputs | sed -e '/^63.81.88/s/32$/31/' | aggregate

If you need a bigger hammer, sed -e 's/32$/31/' is your friend. :)
Re: aggregation tool that allows a bit of fuzz to aggregating ? [ In reply to ]
I guess something like this... maybe? Surely someone has already done this
much better, but I thought it might be a fun puzzle.

# Let's call it aggregate.py. You should test/validate this and not trust
it at all because I don't. It does look like it works, but I can't promise
anything like that. This was "for fun." For me in my world, it's not a
problem that needs solving, but if it helps someone, that'd be pretty
cool. No follow-up questions, please.

./aggregate.py gen 100000 ips.txt # Make up some random IPs for testing
./aggregate.py aggregate 2 ips.txt # Aggregate... second argument is the
"gap", third is the filename...

Most are still going to be /32s.
Some might look like this - maybe even bigger:
27.151.199.176/29
33.58.49.184/29
40.167.88.192/29
63.81.88.112/28 # This is your example set of IPs with a gap (difference)
of 2.
200.42.160.124/30

"max gap" is the distance between IP addresses that can be clustered... an
improvement might include "coverage" - a parameter indicating how many IPs
must appear (ratio) in a cluster to create the aggregate (more meaningful
with bigger gaps).

#!/your/path/to/python
import random
import sys

def inet_aton(ip_string):
octs = ip_string.split('.')
n = int(int(octs[0]) << 24) + int(int(octs[1]) << 16) +
int(int(octs[2]) << 8) + int(octs[3])
return n

def inet_ntoa(ip):
octs = ( ip >> 24, (ip >> 16 & 255), (ip >> 8) & 255, ip & 255 )
return str(octs[0]) + "." + str(octs[1]) + "." + str(octs[2]) + "."
+ str(octs[3])

def gen_ips(num):
ips = []
for x in range(num):
ips.append(inet_ntoa(random.randint(0,pow(2,32)-1)))
# To make sure we have at least SOME nearlyconsecutive IPs...
ips +=
"63.81.88.116,63.81.88.118,63.81.88.120,63.81.88.122,63.81.88.124,63.81.88.126".split(",")
# I added your example IPs.
return ips

def write_random_ips(num,fname):
ips = gen_ips(int(num))
f = open(fname,'w')
for ip in ips:
f.write(ip+'\n')
f.close()

def read_ips(fname):
return open(fname,'r').read(99999999).split('\n')

class Cluster():
def __init__(self):
self.ips = []
def add_ip(self,ip):
self.ips.append(ip)

def find_common_bits(ipa,ipb):
for bits in range(0,32):
mask = pow(2,32)-1 << bits & (pow(2,32)-1)

if ipa & mask == ipb & mask:
return 32-bits
else:
pass # print(f"{ipa} & (pow(2,{bits})-1) == {ipa &
(pow(2,bits)-1)} ==!=== {ipb} & (pow(2,{bits})-1) == {ipb &
(pow(2,bits)-1)}")

if len(sys.argv) == 4 and sys.argv[1] == "generate":
write_random_ips(sys.argv[2],sys.argv[3])
elif len(sys.argv) == 4 and sys.argv[1] == "aggregate": # TODO: Let's
imagine a "coverage" field that augments the max_gap field... does the
prefix cover too many IPs?
max_gap = int(sys.argv[2])
fname = sys.argv[3]

ips = [ inet_aton(ip) for ip in read_ips(fname) if ip!='' ] # ... it'd
be a good idea to make sure it looks like an IP. Oh, this only does IPv4
btw.

ips.sort()

clusters=[Cluster()] # Add first (empty) cluster.. is this necessary?
Who cares, moving on....
last_ip=None
for ip in ips:
if last_ip != None:
#print(f"Gap of {ip-last_ip} between {ip} and {last_ip}...
{inet_ntoa(ip)} / {inet_ntoa(last_ip)}")
if ip - last_ip <= max_gap:
#print(f"Gap of {ip-last_ip} between {ip} and {last_ip}...")
clusters[-1].add_ip(ip)
else:
cluster=Cluster()
cluster.add_ip(ip)
clusters.append(cluster)
last_ip = ip

for cluster in clusters:
if len(cluster.ips) == 0:
continue
if len(cluster.ips) > 1:
first_ip=cluster.ips[0]
last_ip=cluster.ips[-1]
num_bits = find_common_bits(first_ip,last_ip)
mask = pow(2,32)-1 << (32-num_bits) & (pow(2,32)-1)
network = first_ip & mask
print(f"{inet_ntoa(network)}/{num_bits}")
else:
print(f"{inet_ntoa(cluster.ips[0])}/32")
else:
print("Usage:")
print("{0} generate [number of IPs] [file name] # Generate specified
number of IPs, save to [file name]")
print("{0} aggregate [max gap] [file name] # Aggregate prefixes based
on overlapping subnets/IPs per the max gap permitted...")
Re: aggregation tool that allows a bit of fuzz to aggregating ? [ In reply to ]
We use Perl to accomplish this kind of thing.

We blackhole /32s, when we have “enough” of them in the same /24, we remove the /32s after inserting a covering /24. This is a 4 line script, along the same lines of the sed and python suggestions.

Our threshold is pretty low. If we see 4 simultaneous bad actors from the same /24 it’s gone. But we have a very fair process of putting them back into use, think fail2ban.

Best,

Deepak


On Jun 14, 2021, at 3:51 AM, Chris Hartley <hartleyc@gmail.com> wrote:

?
I guess something like this... maybe? Surely someone has already done this much better, but I thought it might be a fun puzzle.

# Let's call it aggregate.py. You should test/validate this and not trust it at all because I don't. It does look like it works, but I can't promise anything like that. This was "for fun." For me in my world, it's not a problem that needs solving, but if it helps someone, that'd be pretty cool. No follow-up questions, please.

./aggregate.py gen 100000 ips.txt # Make up some random IPs for testing
./aggregate.py aggregate 2 ips.txt # Aggregate... second argument is the "gap", third is the filename...

Most are still going to be /32s.
Some might look like this - maybe even bigger:
27.151.199.176/29<http://27.151.199.176/29>
33.58.49.184/29<http://33.58.49.184/29>
40.167.88.192/29<http://40.167.88.192/29>
63.81.88.112/28<http://63.81.88.112/28> # This is your example set of IPs with a gap (difference) of 2.
200.42.160.124/30<http://200.42.160.124/30>

"max gap" is the distance between IP addresses that can be clustered... an improvement might include "coverage" - a parameter indicating how many IPs must appear (ratio) in a cluster to create the aggregate (more meaningful with bigger gaps).

#!/your/path/to/python
import random
import sys

def inet_aton(ip_string):
octs = ip_string.split('.')
n = int(int(octs[0]) << 24) + int(int(octs[1]) << 16) + int(int(octs[2]) << 8) + int(octs[3])
return n

def inet_ntoa(ip):
octs = ( ip >> 24, (ip >> 16 & 255), (ip >> 8) & 255, ip & 255 )
return str(octs[0]) + "." + str(octs[1]) + "." + str(octs[2]) + "." + str(octs[3])

def gen_ips(num):
ips = []
for x in range(num):
ips.append(inet_ntoa(random.randint(0,pow(2,32)-1)))
# To make sure we have at least SOME nearlyconsecutive IPs...
ips += "63.81.88.116,63.81.88.118,63.81.88.120,63.81.88.122,63.81.88.124,63.81.88.126".split(",") # I added your example IPs.
return ips

def write_random_ips(num,fname):
ips = gen_ips(int(num))
f = open(fname,'w')
for ip in ips:
f.write(ip+'\n')
f.close()

def read_ips(fname):
return open(fname,'r').read(99999999).split('\n')

class Cluster():
def __init__(self):
self.ips = []
def add_ip(self,ip):
self.ips.append(ip)

def find_common_bits(ipa,ipb):
for bits in range(0,32):
mask = pow(2,32)-1 << bits & (pow(2,32)-1)

if ipa & mask == ipb & mask:
return 32-bits
else:
pass # print(f"{ipa} & (pow(2,{bits})-1) == {ipa & (pow(2,bits)-1)} ==!=== {ipb} & (pow(2,{bits})-1) == {ipb & (pow(2,bits)-1)}")

if len(sys.argv) == 4 and sys.argv[1] == "generate":
write_random_ips(sys.argv[2],sys.argv[3])
elif len(sys.argv) == 4 and sys.argv[1] == "aggregate": # TODO: Let's imagine a "coverage" field that augments the max_gap field... does the prefix cover too many IPs?
max_gap = int(sys.argv[2])
fname = sys.argv[3]

ips = [ inet_aton(ip) for ip in read_ips(fname) if ip!='' ] # ... it'd be a good idea to make sure it looks like an IP. Oh, this only does IPv4 btw.

ips.sort()

clusters=[Cluster()] # Add first (empty) cluster.. is this necessary? Who cares, moving on....
last_ip=None
for ip in ips:
if last_ip != None:
#print(f"Gap of {ip-last_ip} between {ip} and {last_ip}... {inet_ntoa(ip)} / {inet_ntoa(last_ip)}")
if ip - last_ip <= max_gap:
#print(f"Gap of {ip-last_ip} between {ip} and {last_ip}...")
clusters[-1].add_ip(ip)
else:
cluster=Cluster()
cluster.add_ip(ip)
clusters.append(cluster)
last_ip = ip

for cluster in clusters:
if len(cluster.ips) == 0:
continue
if len(cluster.ips) > 1:
first_ip=cluster.ips[0]
last_ip=cluster.ips[-1]
num_bits = find_common_bits(first_ip,last_ip)
mask = pow(2,32)-1 << (32-num_bits) & (pow(2,32)-1)
network = first_ip & mask
print(f"{inet_ntoa(network)}/{num_bits}")
else:
print(f"{inet_ntoa(cluster.ips[0])}/32")
else:
print("Usage:")
print("{0} generate [number of IPs] [file name] # Generate specified number of IPs, save to [file name]")
print("{0} aggregate [max gap] [file name] # Aggregate prefixes based on overlapping subnets/IPs per the max gap permitted...")