diff --git a/decoder/packet/utils.go b/decoder/packet/utils.go index 95b9a7d7..e4fc85b4 100644 --- a/decoder/packet/utils.go +++ b/decoder/packet/utils.go @@ -20,12 +20,10 @@ package packet import ( - "bytes" "crypto/md5" "encoding/binary" "encoding/hex" "fmt" - "math" "net" "os" "reflect" @@ -45,6 +43,7 @@ import ( "github.com/dreadl0ck/netcap/defaults" "github.com/dreadl0ck/netcap" + byteentropy "github.com/dreadl0ck/netcap/internal/entropy" "github.com/dreadl0ck/netcap/internal/table" netio "github.com/dreadl0ck/netcap/io" "github.com/dreadl0ck/netcap/types" @@ -404,19 +403,9 @@ func ShowDecoders(verbose bool) { } } -// entropy returns the shannon entropy value -// https://rosettacode.org/wiki/Entropy#Go -func entropy(data []byte) (entropy float64) { - if len(data) == 0 { - return 0 - } - for i := range 256 { - px := float64(bytes.Count(data, []byte{byte(i)})) / float64(len(data)) - if px > 0 { - entropy += -px * math.Log2(px) - } - } - return entropy +// entropy returns the Shannon entropy in bits per byte. +func entropy(data []byte) float64 { + return byteentropy.Bytes(data) } const dot = byte('.') diff --git a/decoder/stream/file/file_analysis.go b/decoder/stream/file/file_analysis.go index 62408329..611a1490 100644 --- a/decoder/stream/file/file_analysis.go +++ b/decoder/stream/file/file_analysis.go @@ -21,9 +21,10 @@ package file import ( "bytes" - "math" "path/filepath" "strings" + + "github.com/dreadl0ck/netcap/internal/entropy" ) // FileAnalysis contains security analysis results for a file @@ -142,28 +143,7 @@ func AnalyzeFile(content []byte, filename string) *FileAnalysis { // Returns a value between 0 (uniform) and 8 (random) // Values > 7.0 typically indicate encrypted or compressed content func calculateEntropy(data []byte) float64 { - if len(data) == 0 { - return 0 - } - - // Count byte frequencies - freq := make([]int, 256) - for _, b := range data { - freq[b]++ - } - - // Calculate entropy - var entropy float64 - dataLen := float64(len(data)) - - for _, count := range freq { - if count > 0 { - p := float64(count) / dataLen - entropy -= p * math.Log2(p) - } - } - - return entropy + return entropy.Bytes(data) } // detectFileTypeFromMagic detects file type from magic bytes diff --git a/decoder/stream/protobuf/protobuf.go b/decoder/stream/protobuf/protobuf.go index ebb1b858..57234282 100644 --- a/decoder/stream/protobuf/protobuf.go +++ b/decoder/stream/protobuf/protobuf.go @@ -37,6 +37,7 @@ import ( decoderconfig "github.com/dreadl0ck/netcap/decoder/config" "github.com/dreadl0ck/netcap/decoder/core" streamutils "github.com/dreadl0ck/netcap/decoder/stream/utils" + "github.com/dreadl0ck/netcap/internal/entropy" logging "github.com/dreadl0ck/netcap/internal/logger" "github.com/dreadl0ck/netcap/types" ) @@ -602,24 +603,7 @@ func IsPrintable(data []byte) bool { // CalculateEntropy computes Shannon entropy of the data in bits. func CalculateEntropy(data []byte) float64 { - if len(data) == 0 { - return 0 - } - - freq := make(map[byte]int) - for _, b := range data { - freq[b]++ - } - - entropy := 0.0 - length := float64(len(data)) - - for _, count := range freq { - p := float64(count) / length - entropy -= p * math.Log2(p) - } - - return entropy + return entropy.Bytes(data) } // DetectMessageType classifies a decoded message based on field patterns. diff --git a/internal/entropy/entropy.go b/internal/entropy/entropy.go new file mode 100644 index 00000000..ba690664 --- /dev/null +++ b/internal/entropy/entropy.go @@ -0,0 +1,26 @@ +// Package entropy calculates Shannon entropy over byte distributions. +package entropy + +import "math" + +// Bytes returns Shannon entropy in bits per byte, or zero for empty input. +func Bytes(data []byte) float64 { + if len(data) == 0 { + return 0 + } + + var counts [256]int + for _, b := range data { + counts[b]++ + } + + var result float64 + length := float64(len(data)) + for _, count := range counts { + if count > 0 { + p := float64(count) / length + result -= p * math.Log2(p) + } + } + return result +} diff --git a/internal/entropy/entropy_test.go b/internal/entropy/entropy_test.go new file mode 100644 index 00000000..96c05545 --- /dev/null +++ b/internal/entropy/entropy_test.go @@ -0,0 +1,159 @@ +package entropy + +import ( + "bytes" + "fmt" + "math" + "math/rand" + "testing" +) + +const entropyTolerance = 1e-12 + +var entropySink float64 + +func TestBytes(t *testing.T) { + all := make([]byte, 256) + for i := range all { + all[i] = byte(i) + } + for _, tc := range []struct { + name string + data []byte + want float64 + }{ + {"nil", nil, 0}, + {"empty", []byte{}, 0}, + {"singleton", []byte{255}, 0}, + {"repeated", bytes.Repeat([]byte{42}, 4096), 0}, + {"balanced", bytes.Repeat([]byte{0, 255}, 128), 1}, + {"all256", all, 8}, + {"skewed", []byte{0, 0, 0, 255}, -0.75*math.Log2(0.75) - 0.25*math.Log2(0.25)}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := Bytes(tc.data); math.IsNaN(got) || math.Abs(got-tc.want) > entropyTolerance { + t.Fatalf("Bytes = %.17g, want %.17g", got, tc.want) + } + if allocs := testing.AllocsPerRun(100, func() { entropySink = Bytes(tc.data) }); allocs != 0 { + t.Errorf("Bytes allocated %g times, want zero", allocs) + } + }) + } +} + +func TestBytesRandomized(t *testing.T) { + rng := rand.New(rand.NewSource(1)) + for i := range 200 { + data := make([]byte, rng.Intn(64*1024+1)) + alphabet := 1 + rng.Intn(256) + for j := range data { + data[j] = byte(rng.Intn(alphabet)) + } + got, want := Bytes(data), repeatedCountEntropy(data) + if math.IsNaN(got) || math.Abs(got-want) > entropyTolerance { + t.Fatalf("case %d (size %d, alphabet %d): Bytes = %.17g, want %.17g", i, len(data), alphabet, got, want) + } + } +} + +func FuzzBytes(f *testing.F) { + f.Add([]byte{}) + f.Add([]byte{255}) + f.Add(bytes.Repeat([]byte{42}, 512)) + f.Add([]byte{0, 255, 0, 255}) + f.Add([]byte{0, 0, 0, 255}) + all := make([]byte, 256) + for i := range all { + all[i] = byte(i) + } + f.Add(all) + f.Add(entropyData(1500, "text")) + f.Add(entropyData(64*1024, "random")) + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > 64*1024 { + data = data[:64*1024] + } + got, want := Bytes(data), repeatedCountEntropy(data) + if math.IsNaN(got) || math.Abs(got-want) > entropyTolerance { + t.Fatalf("size %d: Bytes = %.17g, want %.17g", len(data), got, want) + } + }) +} + +// repeatedCountEntropy is the old packet implementation and differential reference. +func repeatedCountEntropy(data []byte) (entropy float64) { + if len(data) == 0 { + return 0 + } + for i := range 256 { + p := float64(bytes.Count(data, []byte{byte(i)})) / float64(len(data)) + if p > 0 { + entropy += -p * math.Log2(p) + } + } + return entropy +} + +func protobufMapEntropy(data []byte) float64 { + if len(data) == 0 { + return 0 + } + freq := make(map[byte]int) + for _, b := range data { + freq[b]++ + } + entropy := 0.0 + length := float64(len(data)) + for _, count := range freq { + p := float64(count) / length + entropy -= p * math.Log2(p) + } + return entropy +} + +func entropyData(size int, distribution string) []byte { + data := make([]byte, size) + switch distribution { + case "repeated": + for i := range data { + data[i] = 42 + } + case "text": + const text = "GET /index.html HTTP/1.1\r\nHost: example.com\r\nContent-Type: text/plain\r\n\r\nThe quick brown fox jumps over the lazy dog.\n" + for i := range data { + data[i] = text[i%len(text)] + } + case "random": + rng := rand.New(rand.NewSource(1)) + for i := range data { + data[i] = byte(rng.Intn(256)) + } + default: + panic("unknown entropy distribution: " + distribution) + } + return data +} + +func BenchmarkBytes(b *testing.B) { + for _, size := range []int{64, 512, 1500, 16384, 1 << 20} { + for _, distribution := range []string{"repeated", "text", "random"} { + data := entropyData(size, distribution) + for _, impl := range []struct { + name string + fn func([]byte) float64 + }{ + {"Bytes", Bytes}, + {"OldPacketRepeatedCount", repeatedCountEntropy}, + {"OldProtobufMap", protobufMapEntropy}, + } { + b.Run(fmt.Sprintf("%d/%s/%s", size, distribution, impl.name), func(b *testing.B) { + b.SetBytes(int64(len(data))) + b.ReportAllocs() + for b.Loop() { + entropySink = impl.fn(data) + } + }) + } + } + } +} diff --git a/internal/entropy/experiment.go b/internal/entropy/experiment.go new file mode 100644 index 00000000..b83e020e --- /dev/null +++ b/internal/entropy/experiment.go @@ -0,0 +1,43 @@ +//go:build entropyexperiment + +package entropy + +import "math" + +// Four banks reduce repeated-byte dependencies but add setup and merging work. +type experimentBanks [4][256]int + +func histogramGo4(data []byte, banks *experimentBanks) { + for len(data) >= 4 { + banks[0][data[0]]++ + banks[1][data[1]]++ + banks[2][data[2]]++ + banks[3][data[3]]++ + data = data[4:] + } + for _, v := range data { + banks[0][v]++ + } +} + +func experimentEntropy(banks *experimentBanks, size int) float64 { + var result float64 + for i, count := range banks[0] { + count += banks[1][i] + banks[2][i] + banks[3][i] + if count > 0 { + p := float64(count) / float64(size) + result -= p * math.Log2(p) + } + } + return result +} + +// BytesGo4 evaluates the portable four-bank experiment, including setup and logs. +func BytesGo4(data []byte) float64 { + if len(data) == 0 { + return 0 + } + var banks experimentBanks + histogramGo4(data, &banks) + return experimentEntropy(&banks, len(data)) +} diff --git a/internal/entropy/experiment_arm64.go b/internal/entropy/experiment_arm64.go new file mode 100644 index 00000000..7cb7236b --- /dev/null +++ b/internal/entropy/experiment_arm64.go @@ -0,0 +1,18 @@ +//go:build entropyexperiment && !purego + +package entropy + +const experimentASMName = "ASM4" + +//go:noescape +func histogramARM64(data []byte, banks *experimentBanks) + +// BytesARM64 evaluates scalar four-bank ARM64 assembly, including setup and logs. +func BytesARM64(data []byte) float64 { + if len(data) == 0 { + return 0 + } + var banks experimentBanks + histogramARM64(data, &banks) + return experimentEntropy(&banks, len(data)) +} diff --git a/internal/entropy/experiment_arm64.s b/internal/entropy/experiment_arm64.s new file mode 100644 index 00000000..d7dfe995 --- /dev/null +++ b/internal/entropy/experiment_arm64.s @@ -0,0 +1,46 @@ +//go:build entropyexperiment && !purego + +#include "textflag.h" + +// Scalar, four independent banks. Only load bytes known to be in the slice. +// Interleave updates: batching counter loads regresses repeated-byte throughput. +TEXT ·histogramARM64(SB), NOSPLIT, $0-32 + MOVD data_base+0(FP), R0 + MOVD data_len+8(FP), R1 + MOVD banks+24(FP), R2 + ADD $2048, R2, R3 + ADD $2048, R3, R4 + ADD $2048, R4, R5 +loop: + CMP $4, R1 + BLT tail + MOVBU (R0), R6 + MOVD (R2)(R6<<3), R10 + ADD $1, R10 + MOVD R10, (R2)(R6<<3) + MOVBU 1(R0), R7 + MOVD (R3)(R7<<3), R11 + ADD $1, R11 + MOVD R11, (R3)(R7<<3) + MOVBU 2(R0), R8 + MOVD (R4)(R8<<3), R12 + ADD $1, R12 + MOVD R12, (R4)(R8<<3) + MOVBU 3(R0), R9 + MOVD (R5)(R9<<3), R13 + ADD $1, R13 + MOVD R13, (R5)(R9<<3) + ADD $4, R0 + SUB $4, R1 + B loop +tail: + CBZ R1, done + MOVBU (R0), R6 + MOVD (R2)(R6<<3), R10 + ADD $1, R10 + MOVD R10, (R2)(R6<<3) + ADD $1, R0 + SUB $1, R1 + B tail +done: + RET diff --git a/internal/entropy/experiment_fallback.go b/internal/entropy/experiment_fallback.go new file mode 100644 index 00000000..6fad12ba --- /dev/null +++ b/internal/entropy/experiment_fallback.go @@ -0,0 +1,12 @@ +//go:build entropyexperiment && (!arm64 || purego) + +package entropy + +const experimentASMName = "GoFallback" + +func histogramARM64(data []byte, banks *experimentBanks) { + histogramGo4(data, banks) +} + +// BytesARM64 uses portable Go when ARM64 assembly is unavailable or disabled. +func BytesARM64(data []byte) float64 { return BytesGo4(data) } diff --git a/internal/entropy/experiment_guard_test.go b/internal/entropy/experiment_guard_test.go new file mode 100644 index 00000000..b40faaf7 --- /dev/null +++ b/internal/entropy/experiment_guard_test.go @@ -0,0 +1,26 @@ +//go:build entropyexperiment && darwin && arm64 && !purego + +package entropy + +import ( + "syscall" + "testing" +) + +func TestExperimentGuardPage(t *testing.T) { + page := syscall.Getpagesize() + data, err := syscall.Mmap(-1, 0, 2*page, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANON|syscall.MAP_PRIVATE) + if err != nil { + t.Fatal(err) + } + defer syscall.Munmap(data) + for i := range data[:page] { + data[i] = byte(i) + } + if err := syscall.Mprotect(data[page:], syscall.PROT_NONE); err != nil { + t.Fatal(err) + } + for n := 0; n <= page; n++ { + checkExperiment(t, data[page-n:page:page]) + } +} diff --git a/internal/entropy/experiment_pcap_test.go b/internal/entropy/experiment_pcap_test.go new file mode 100644 index 00000000..434befc6 --- /dev/null +++ b/internal/entropy/experiment_pcap_test.go @@ -0,0 +1,101 @@ +//go:build entropyexperiment && (darwin || linux) + +package entropy + +import ( + "io" + "os" + "syscall" + "testing" + + "github.com/gopacket/gopacket" + "github.com/gopacket/gopacket/layers" + "github.com/gopacket/gopacket/pcapgo" +) + +// BenchmarkCapturePayloads replays the entropy inputs of four packet decoders. +// Set NETCAP_ENTROPY_PCAP to a pcapng; mixed per-packet link types are handled. +// Reading and packet decoding are not timed. +func BenchmarkCapturePayloads(b *testing.B) { + path := os.Getenv("NETCAP_ENTROPY_PCAP") + if path == "" { + b.Skip("set NETCAP_ENTROPY_PCAP to a pcapng") + } + f, err := os.Open(path) + if err != nil { + b.Fatal(err) + } + defer f.Close() + opts := pcapgo.DefaultNgReaderOptions + opts.WantMixedLinkType = true + r, err := pcapgo.NewNgReader(f, opts) + if err != nil { + b.Fatal(err) + } + var payloads [][]byte + var total int64 + var large, packets int + for { + data, ci, err := r.ReadPacketData() + if err == io.EOF { + break + } + if err != nil { + b.Fatal(err) + } + packets++ + iface, err := r.Interface(ci.InterfaceIndex) + if err != nil { + b.Fatal(err) + } + packet := gopacket.NewPacket(data, iface.LinkType, gopacket.Default) + for _, layer := range packet.Layers() { + switch layer.LayerType() { + case layers.LayerTypeEthernet, layers.LayerTypeIPv4, layers.LayerTypeTCP, layers.LayerTypeUDP: + payload := layer.LayerPayload() + payloads = append(payloads, payload) + total += int64(len(payload)) + if len(payload) >= 16384 { + large++ + } + } + } + } + if len(payloads) == 0 { + b.Fatal("capture has no matching packet layers") + } + b.Logf("%d packets, %d entropy inputs, %d bytes, %d inputs >= 16 KiB", packets, len(payloads), total, large) + for _, impl := range []struct { + name string + fn func([]byte) float64 + }{ + {"Bytes", Bytes}, + {"OldPacketRepeatedCount", repeatedCountEntropy}, + {"OldProtobufMap", protobufMapEntropy}, + {"Go4", BytesGo4}, + {experimentASMName, BytesARM64}, + } { + b.Run(impl.name, func(b *testing.B) { + b.SetBytes(total) + b.ReportAllocs() + var before, after syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &before); err != nil { + b.Fatal(err) + } + for b.Loop() { + var sum float64 + for _, payload := range payloads { + sum += impl.fn(payload) + } + entropySink = sum + } + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &after); err != nil { + b.Fatal(err) + } + cpu := after.Utime.Nano() + after.Stime.Nano() - before.Utime.Nano() - before.Stime.Nano() + b.ReportMetric(float64(cpu)/float64(b.N), "cpu-ns/op") + b.ReportMetric(float64(large), "large-inputs/op") + b.ReportMetric(float64(len(payloads)), "inputs/op") + }) + } +} diff --git a/internal/entropy/experiment_test.go b/internal/entropy/experiment_test.go new file mode 100644 index 00000000..1684a807 --- /dev/null +++ b/internal/entropy/experiment_test.go @@ -0,0 +1,116 @@ +//go:build entropyexperiment + +package entropy + +import ( + "bytes" + "fmt" + "math" + "math/rand" + "testing" +) + +func checkExperiment(t *testing.T, data []byte) { + t.Helper() + var want [256]int + var goBanks, asmBanks experimentBanks + for _, v := range data { + want[v]++ + } + histogramGo4(data, &goBanks) + histogramARM64(data, &asmBanks) + if goBanks != asmBanks { + t.Fatalf("size %d: bank mismatch", len(data)) + } + for i, count := range want { + if got := asmBanks[0][i] + asmBanks[1][i] + asmBanks[2][i] + asmBanks[3][i]; got != count { + t.Fatalf("size %d byte %d: count %d, want %d", len(data), i, got, count) + } + } + wantEntropy := math.Float64bits(Bytes(data)) + for _, fn := range []func([]byte) float64{BytesGo4, BytesARM64} { + if got := fn(data); math.Float64bits(got) != wantEntropy { + t.Fatalf("size %d: entropy %.17g, want %.17g", len(data), got, math.Float64frombits(wantEntropy)) + } + } +} + +func TestExperiment(t *testing.T) { + checkExperiment(t, nil) + for v := range 256 { + for _, n := range []int{1, 3, 4, 15, 16, 31, 32, 63, 64, 511, 512, 513, 1500, 16387} { + checkExperiment(t, bytes.Repeat([]byte{byte(v)}, n)) + } + } + rng := rand.New(rand.NewSource(42)) + data := make([]byte, 65536+32) + for trial := range 300 { + _, _ = rng.Read(data) + n := rng.Intn(65537) + if trial < 128 { + n = trial + } + for offset := range 16 { + checkExperiment(t, data[offset:offset+n:offset+n]) + } + } + for _, distribution := range []string{"repeated", "text", "random"} { + checkExperiment(t, entropyData(1<<20, distribution)) + } + // Nonzero, wide counters catch truncation and verify additive kernel semantics. + var goBanks experimentBanks + for bank := range goBanks { + for v := range goBanks[bank] { + goBanks[bank][v] = int(^uint(0)>>16) + bank + v + } + } + asmBanks := goBanks + histogramGo4(data, &goBanks) + histogramARM64(data, &asmBanks) + if goBanks != asmBanks { + t.Fatal("nonzero bank mismatch") + } + for _, n := range []int{0, 1, 64, 512, 1500, 16384, 1 << 20} { + data := entropyData(n, "random") + for _, fn := range []func([]byte) float64{BytesGo4, BytesARM64} { + if allocs := testing.AllocsPerRun(20, func() { entropySink = fn(data) }); allocs != 0 { + t.Fatalf("size %d: %g allocations", n, allocs) + } + } + } +} + +func FuzzExperiment(f *testing.F) { + for _, n := range []int{0, 1, 3, 4, 15, 16, 31, 32, 63, 64, 512, 1500, 16384} { + f.Add(entropyData(n, "random")) + f.Add(entropyData(n, "repeated")) + } + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > 1<<20 { + data = data[:1<<20] + } + checkExperiment(t, data) + }) +} + +// Full entropy: all candidates include stack zeroing, merging, and exact logs. +// go test -tags entropyexperiment -run '^$' -bench '^BenchmarkExperiment$' -benchtime=300ms -count=3 -cpu=1 +func BenchmarkExperiment(b *testing.B) { + for _, n := range []int{0, 1, 15, 16, 31, 32, 63, 64, 512, 1500, 16384, 1 << 20} { + for _, distribution := range []string{"repeated", "text", "random"} { + data := entropyData(n, distribution) + for _, impl := range []struct { + name string + fn func([]byte) float64 + }{{"Bytes", Bytes}, {"Go4", BytesGo4}, {experimentASMName, BytesARM64}} { + b.Run(fmt.Sprintf("%d/%s/%s", n, distribution, impl.name), func(b *testing.B) { + b.SetBytes(int64(n)) + b.ReportAllocs() + for b.Loop() { + entropySink = impl.fn(data) + } + }) + } + } + } +}