From c3c93c74f733ade497b76d75649c35a5207cf33c Mon Sep 17 00:00:00 2001 From: v-byte-cpu <65545655+v-byte-cpu@users.noreply.github.com> Date: Fri, 25 Jun 2021 00:32:01 +0300 Subject: [PATCH] feature: randomized IP iterator (#89) --- README.md | 1 + command/root.go | 2 + pkg/scan/range.go | 281 +++++++++++++++++++++++++++++++++++++++ pkg/scan/range_test.go | 152 +++++++++++++++++++++ pkg/scan/request.go | 28 +++- pkg/scan/request_test.go | 41 ++++-- 6 files changed, 490 insertions(+), 15 deletions(-) create mode 100644 pkg/scan/range.go create mode 100644 pkg/scan/range_test.go diff --git a/README.md b/README.md index 7d92ed6..21a5f37 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ The goal of this project is to create the fastest network scanner with clean and * **SOCKS5 scan**: Detect live SOCKS5 proxies by scanning ip range or list of ip/port pairs from a file * **Docker scan**: Detect open Docker daemons listening on TCP ports and get information about the docker node * **Elasticsearch scan**: Detect open Elasticsearch nodes and pull out cluster information with all index names + * **Randomized iteration** over IP addresses using finite cyclic multiplicative groups * **JSON output support**: sx is designed specifically for convenient automatic processing of results ## 📦 Install diff --git a/command/root.go b/command/root.go index bc26219..40cd4e9 100644 --- a/command/root.go +++ b/command/root.go @@ -2,6 +2,7 @@ package command import ( "context" + "math/rand" "os" "sync" "time" @@ -15,6 +16,7 @@ import ( ) func Main(version string) { + rand.Seed(time.Now().Unix()) if err := newRootCmd(version).Execute(); err != nil { os.Exit(1) } diff --git a/pkg/scan/range.go b/pkg/scan/range.go new file mode 100644 index 0000000..f0487bf --- /dev/null +++ b/pkg/scan/range.go @@ -0,0 +1,281 @@ +package scan + +import ( + "errors" + "fmt" + "math/big" + "math/rand" + "sort" +) + +var errRangeSize = errors.New("invalid range size") + +// We will pick the first cyclic group from this list that is +// larger than the range size +var cyclicGroups = []struct { + // Prime number for (Z/pZ)* multiplicative group + P int64 + // Cyclic group generator + G int64 + // Number coprime with P-1 + N int64 +}{ + { + P: 3, // 2^1 + 1 + G: 2, + N: 1, + }, + { + P: 5, // 2^2 + 1 + G: 2, + N: 1, + }, + { + P: 11, // 2^3 + 3 + G: 2, + N: 3, + }, + { + P: 17, // 2^4 + 1 + G: 3, + N: 3, + }, + { + P: 37, // 2^5 + 5 + G: 2, + N: 5, + }, + { + P: 67, // 2^6 + 3 + G: 2, + N: 5, + }, + { + P: 131, // 2^7 + 3 + G: 2, + N: 3, + }, + { + P: 257, // 2^8 + 1 + G: 3, + N: 3, + }, + { + P: 523, // 2^9 + 11 + G: 2, + N: 5, + }, + { + P: 1031, // 2^10 + 7 + G: 21, + N: 3, + }, + { + P: 2053, // 2^11 + 5 + G: 2, + N: 5, + }, + { + P: 4099, // 2^12 + 3 + G: 2, + N: 5, + }, + { + P: 8219, // 2^13 + 27 + G: 2, + N: 3, + }, + { + P: 16421, // 2^14 + 37 + G: 2, + N: 3, + }, + { + P: 32771, // 2^15 + 3 + G: 2, + N: 3, + }, + { + P: 65539, // 2^16 + 3 + G: 2, + N: 5, + }, + { + P: 131101, // 2^17 + 29 + G: 17, + N: 7, + }, + { + P: 262147, // 2^18 + 3 + G: 2, + N: 5, + }, + { + P: 524309, // 2^19 + 21 + G: 2, + N: 3, + }, + { + P: 1048589, // 2^20 + 13 + G: 2, + N: 3, + }, + { + P: 2097211, // 2^21 + 59 + G: 2, + N: 7, + }, + { + P: 4194371, // 2^22 + 67 + G: 2, + N: 3, + }, + { + P: 8388619, // 2^23 + 11 + G: 2, + N: 5, + }, + { + P: 16777259, // 2^24 + 43 + G: 2, + N: 5, + }, + { + P: 33554467, // 2^25 + 35 + G: 2, + N: 5, + }, + { + P: 67108933, // 2^26 + 69 + G: 2, + N: 5, + }, + { + P: 134217773, // 2^27 + 45 + G: 2, + N: 5, + }, + { + P: 268435459, // 2^28 + 3 + G: 2, + N: 5, + }, + { + P: 536871019, // 2^29 + 107 + G: 2, + N: 5, + }, + { + P: 1073741827, // 2^30 + 3 + G: 2, + N: 5, + }, + { + P: 2147483659, // 2^31 + 11 + G: 2, + N: 5, + }, + { + P: 4294967357, // 2^32 + 61 + G: 2, + N: 5, + }, +} + +// newRangeIterator creates a pseudo-random iterator for +// integer range [1..n]. Each integer is traversed exactly once. +func newRangeIterator(n int64) (*rangeIterator, error) { + // Here we apply cyclic groups + // (Z/pZ)* is a multiplicative group if p is a prime number + // also (Z/pZ)* is a cyclic group, to understand this fact I recommend to read + // "When Is the Multiplicative Group Modulo n Cyclic?" paper by Aryeh Zax + if n <= 0 { + return nil, errRangeSize + } + + // find first cyclic group that is larger than n + idx := sort.Search(len(cyclicGroups), func(i int) bool { + return cyclicGroups[i].P > n + }) + if idx == len(cyclicGroups) { + return nil, errRangeSize + } + cyclic := cyclicGroups[idx] + P, G, N := big.NewInt(cyclic.P), big.NewInt(cyclic.G), big.NewInt(cyclic.N) + + // first of all, we apply group theory facts for cyclic groups: + // 1. Let T be a finite cyclic group of order n. Let G be a generator. Let r be an + // integer != 0, and relatively prime to n. Then (G ** r) is also a generator of T. + // 2. Fermat's little theorem: + // if p is a prime number then for any integer a: (a ** (p-1)) mod p = 1. + // See Chapter 2, Exercise 17 on page 26 and Theorem 4.3 (Lagrange's theorem) + // in the "Undergraduate Algebra" Third Edition by Serge Lang + + // number of elements of (Z/pZ)* is equal to P-1 + // randM is a random integer + randM := big.NewInt(rand.Int63()) + one := big.NewInt(1) + randM.Add(randM, one) + // if N is coprime with P-1 => (N ** randM) is coprime with P-1 + // by Fermat's little theorem: (G ** M) mod P = (G ** (M mod (P-1))) mod P for any integer M + // prepare new group generator: + // G - generator, (N ** randM) is coprime with group order => G = (G ** (N ** randM)) mod P is also a generator + N.Exp(N, randM, big.NewInt(cyclic.P-1)) + G.Exp(G, N, P) + + // select a random element from which to start the iteration: randI = (G ** randM) mod P + randM.SetInt64(rand.Int63()).Add(randM, one) + randI := big.NewInt(0).Exp(G, randM, P) + + it := &rangeIterator{P: P, G: G, + rangeLimit: big.NewInt(n), + I: big.NewInt(0).Set(randI), + startI: big.NewInt(0).Set(randI), + } + + // find a first number I <= n from which to start the iteration + if !it.Next() && n > 1 { + return nil, fmt.Errorf("invalid cyclic group: P = %+v G = %+v N = %+v startI = %+v", + P, G, N, it.startI) + } + it.startI.Set(it.I) + return it, nil +} + +type rangeIterator struct { + // Prime number for (Z/pZ)* multiplicative group + P *big.Int + // Cyclic group generator + G *big.Int + // Current number + I *big.Int + // the number at which the iteration starts + startI *big.Int + + // right boundary of the range + rangeLimit *big.Int + stop bool +} + +func (it *rangeIterator) Next() bool { + if it.stop { + return false + } + for { + // I = (I * G) mod P + it.I.Mul(it.I, it.G) + it.I.Mod(it.I, it.P) + if it.I.Cmp(it.startI) == 0 { + it.stop = true + return false + } + // if i <= rangeLimit + if it.I.Cmp(it.rangeLimit) < 1 { + return true + } + } +} + +func (it *rangeIterator) Int() *big.Int { + return it.I +} diff --git a/pkg/scan/range_test.go b/pkg/scan/range_test.go new file mode 100644 index 0000000..46bca5b --- /dev/null +++ b/pkg/scan/range_test.go @@ -0,0 +1,152 @@ +package scan + +import ( + "math/big" + "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestNewRangeIteratorError(t *testing.T) { + tests := []int64{-1, 0, 1 << 33} + for _, input := range tests { + _, err := newRangeIterator(input) + require.Equal(t, errRangeSize, err, "no error for %d", input) + } +} + +func TestNewRangeIterator(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + n int + }{ + { + name: "1", + n: 1, + }, + { + name: "2", + n: 2, + }, + { + name: "3", + n: 3, + }, + { + name: "4", + n: 4, + }, + { + name: "8", + n: 8, + }, + { + name: "16", + n: 16, + }, + { + name: "17", + n: 17, + }, + { + name: "32", + n: 32, + }, + { + name: "64", + n: 64, + }, + { + name: "128", + n: 128, + }, + { + name: "1 << 8", + n: 1 << 8, + }, + { + name: "1 << 9", + n: 1 << 9, + }, + { + name: "1 << 10", + n: 1 << 10, + }, + { + name: "1 << 11", + n: 1 << 11, + }, + { + name: "1 << 12", + n: 1 << 12, + }, + { + name: "1 << 13", + n: 1 << 13, + }, + { + name: "1 << 14", + n: 1 << 14, + }, + { + name: "1 << 15", + n: 1 << 15, + }, + { + name: "1 << 16", + n: 1 << 16, + }, + } + rand.Seed(time.Now().Unix()) + + for _, vtt := range tests { + tt := vtt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + done := make(chan interface{}) + go func() { + defer close(done) + + it, err := newRangeIterator(int64(tt.n)) + require.NoError(t, err) + bitset := big.NewInt(0) + cnt := 0 + for { + cnt++ + i := int(it.Int().Int64()) + if bitset.Bit(i) == 1 { + require.Fail(t, "number has already been visited", + "number %d, P = %+v G = %+v startI = %+v", i, it.P, it.G, it.startI) + } + bitset.SetBit(bitset, i, 1) + if !it.Next() { + break + } + } + for i := 1; i <= tt.n; i++ { + require.Equal(t, uint(1), bitset.Bit(i), + "number %d is not visited, P = %+v G = %+v startI = %+v", i, it.P, it.G, it.startI) + } + require.Equal(t, tt.n, cnt, "count is not valid") + require.False(t, it.Next()) + }() + waitDone(t, done) + }) + } +} + +func BenchmarkRangeIterator(b *testing.B) { + b.ReportAllocs() + it, err := newRangeIterator(int64(b.N)) + require.NoError(b, err) + for { + if !it.Next() { + break + } + } +} diff --git a/pkg/scan/request.go b/pkg/scan/request.go index ce01289..d671834 100644 --- a/pkg/scan/request.go +++ b/pkg/scan/request.go @@ -8,10 +8,9 @@ import ( "context" "errors" "io" + "math/big" "net" "time" - - "github.com/v-byte-cpu/sx/pkg/ip" ) var ( @@ -99,12 +98,31 @@ func (*ipGenerator) IPs(ctx context.Context, r *Range) (<-chan IPGetter, error) if r.DstSubnet == nil { return nil, ErrSubnet } + ipnet := r.DstSubnet + ones, bits := ipnet.Mask.Size() + it, err := newRangeIterator(1 << (bits - ones)) + if err != nil { + return nil, err + } + + baseIP := big.NewInt(0).SetBytes(ipnet.IP.Mask(ipnet.Mask)) + baseIP.Sub(baseIP, big.NewInt(1)) + out := make(chan IPGetter, 100) go func() { defer close(out) - ipnet := r.DstSubnet - for ipaddr := ipnet.IP.Mask(ipnet.Mask); ipnet.Contains(ipaddr); ip.Inc(ipaddr) { - writeIP(ctx, out, WrapIP(ip.DupIP(ipaddr))) + for { + i := it.Int() + baseIP.Add(baseIP, i) + // TODO IPv6 + ipaddr := baseIP.FillBytes(make([]byte, 4)) + baseIP.Sub(baseIP, i) + + writeIP(ctx, out, WrapIP(ipaddr)) + + if !it.Next() { + return + } } }() return out, nil diff --git a/pkg/scan/request_test.go b/pkg/scan/request_test.go index 6ceb9e8..9d1bd73 100644 --- a/pkg/scan/request_test.go +++ b/pkg/scan/request_test.go @@ -1,11 +1,13 @@ package scan import ( + "bytes" "context" "errors" "io" "io/ioutil" "net" + "sort" "strings" "testing" "time" @@ -296,6 +298,9 @@ func TestIPGenerator(t *testing.T) { } require.NoError(t, err) result := chanToSlice(t, chanIPToGeneric(ips), len(tt.expected)) + sort.Slice(result, func(i, j int) bool { + return bytes.Compare([]byte(result[i].(WrapIP)), []byte(result[j].(WrapIP))) < 1 + }) require.Equal(t, tt.expected, result) }() waitDone(t, done) @@ -343,7 +348,7 @@ func TestIPPortGenerator(t *testing.T) { { name: "OneIpOnePort", input: newScanRange( - withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1), Mask: net.CIDRMask(32, 32)}), + withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1).To4(), Mask: net.CIDRMask(32, 32)}), withPorts([]*PortRange{ { StartPort: 888, @@ -358,7 +363,7 @@ func TestIPPortGenerator(t *testing.T) { { name: "OneIpTwoPorts", input: newScanRange( - withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1), Mask: net.CIDRMask(32, 32)}), + withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1).To4(), Mask: net.CIDRMask(32, 32)}), withPorts([]*PortRange{ { StartPort: 888, @@ -374,7 +379,7 @@ func TestIPPortGenerator(t *testing.T) { { name: "TwoIpsOnePort", input: newScanRange( - withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1), Mask: net.CIDRMask(31, 32)}), + withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1).To4(), Mask: net.CIDRMask(31, 32)}), withPorts([]*PortRange{ { StartPort: 888, @@ -390,7 +395,7 @@ func TestIPPortGenerator(t *testing.T) { { name: "FourIpsOnePort", input: newScanRange( - withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1), Mask: net.CIDRMask(30, 32)}), + withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1).To4(), Mask: net.CIDRMask(30, 32)}), withPorts([]*PortRange{ { StartPort: 888, @@ -408,7 +413,7 @@ func TestIPPortGenerator(t *testing.T) { { name: "TwoIpsTwoPorts", input: newScanRange( - withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1), Mask: net.CIDRMask(31, 32)}), + withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1).To4(), Mask: net.CIDRMask(31, 32)}), withPorts([]*PortRange{ { StartPort: 888, @@ -418,15 +423,15 @@ func TestIPPortGenerator(t *testing.T) { ), expected: []interface{}{ newScanRequest(withDstIP(net.IPv4(192, 168, 0, 0).To4()), withDstPort(888)), - newScanRequest(withDstIP(net.IPv4(192, 168, 0, 1).To4()), withDstPort(888)), newScanRequest(withDstIP(net.IPv4(192, 168, 0, 0).To4()), withDstPort(889)), + newScanRequest(withDstIP(net.IPv4(192, 168, 0, 1).To4()), withDstPort(888)), newScanRequest(withDstIP(net.IPv4(192, 168, 0, 1).To4()), withDstPort(889)), }, }, { name: "OneIpPortOverflow", input: newScanRange( - withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1), Mask: net.CIDRMask(32, 32)}), + withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1).To4(), Mask: net.CIDRMask(32, 32)}), withPorts([]*PortRange{ { StartPort: 65535, @@ -457,6 +462,17 @@ func TestIPPortGenerator(t *testing.T) { } require.NoError(t, err) result := chanToSlice(t, chanPairToGeneric(pairs), len(tt.expected)) + sort.Slice(result, func(i, j int) bool { + req1 := result[i].(*Request) + req2 := result[j].(*Request) + switch bytes.Compare([]byte(req1.DstIP), []byte(req2.DstIP)) { + case -1: + return true + case 1: + return false + } + return req1.DstPort < req2.DstPort + }) require.Equal(t, tt.expected, result) }() waitDone(t, done) @@ -481,7 +497,7 @@ func TestIPRequestGenerator(t *testing.T) { { name: "OneIP", input: newScanRange( - withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1), Mask: net.CIDRMask(32, 32)}), + withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1).To4(), Mask: net.CIDRMask(32, 32)}), ), expected: []interface{}{ newScanRequest(withDstIP(net.IPv4(192, 168, 0, 1).To4())), @@ -490,7 +506,7 @@ func TestIPRequestGenerator(t *testing.T) { { name: "TwoIPs", input: newScanRange( - withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1), Mask: net.CIDRMask(31, 32)}), + withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1).To4(), Mask: net.CIDRMask(31, 32)}), ), expected: []interface{}{ newScanRequest(withDstIP(net.IPv4(192, 168, 0, 0).To4())), @@ -500,7 +516,7 @@ func TestIPRequestGenerator(t *testing.T) { { name: "FourIPs", input: newScanRange( - withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1), Mask: net.CIDRMask(30, 32)}), + withSubnet(&net.IPNet{IP: net.IPv4(192, 168, 0, 1).To4(), Mask: net.CIDRMask(30, 32)}), ), expected: []interface{}{ newScanRequest(withDstIP(net.IPv4(192, 168, 0, 0).To4())), @@ -528,6 +544,11 @@ func TestIPRequestGenerator(t *testing.T) { } require.NoError(t, err) result := chanToSlice(t, chanPairToGeneric(pairs), len(tt.expected)) + sort.Slice(result, func(i, j int) bool { + return bytes.Compare( + []byte(result[i].(*Request).DstIP), + []byte(result[j].(*Request).DstIP)) < 1 + }) require.Equal(t, tt.expected, result) }() waitDone(t, done)