Optimize byte entropy and evaluate ARM64 assembly

decoder/packet/utils.go

@@ -20,12 +20,10 @@
 package packet
 
 import (
-	"bytes"
 	"crypto/md5"
 	"encoding/binary"
 	"encoding/hex"
 	"fmt"
-	"math"
 	"net"
 	"os"
 	"reflect"
@@ -45,6 +43,7 @@ import (
 	"github.com/dreadl0ck/netcap/defaults"
 
 	"github.com/dreadl0ck/netcap"
+	byteentropy "github.com/dreadl0ck/netcap/internal/entropy"
 	"github.com/dreadl0ck/netcap/internal/table"
 	netio "github.com/dreadl0ck/netcap/io"
 	"github.com/dreadl0ck/netcap/types"
@@ -404,19 +403,9 @@ func ShowDecoders(verbose bool) {
 	}
 }
 
-// entropy returns the shannon entropy value
-// https://rosettacode.org/wiki/Entropy#Go
-func entropy(data []byte) (entropy float64) {
-	if len(data) == 0 {
-		return 0
-	}
-	for i := range 256 {
-		px := float64(bytes.Count(data, []byte{byte(i)})) / float64(len(data))
-		if px > 0 {
-			entropy += -px * math.Log2(px)
-		}
-	}
-	return entropy
+// entropy returns the Shannon entropy in bits per byte.
+func entropy(data []byte) float64 {
+	return byteentropy.Bytes(data)
 }
 
 const dot = byte('.')

decoder/stream/file/file_analysis.go

@@ -21,9 +21,10 @@ package file
 
 import (
 	"bytes"
-	"math"
 	"path/filepath"
 	"strings"
+
+	"github.com/dreadl0ck/netcap/internal/entropy"
 )
 
 // FileAnalysis contains security analysis results for a file
@@ -142,28 +143,7 @@ func AnalyzeFile(content []byte, filename string) *FileAnalysis {
 // Returns a value between 0 (uniform) and 8 (random)
 // Values > 7.0 typically indicate encrypted or compressed content
 func calculateEntropy(data []byte) float64 {
-	if len(data) == 0 {
-		return 0
-	}
-
-	// Count byte frequencies
-	freq := make([]int, 256)
-	for _, b := range data {
-		freq[b]++
-	}
-
-	// Calculate entropy
-	var entropy float64
-	dataLen := float64(len(data))
-
-	for _, count := range freq {
-		if count > 0 {
-			p := float64(count) / dataLen
-			entropy -= p * math.Log2(p)
-		}
-	}
-
-	return entropy
+	return entropy.Bytes(data)
 }
 
 // detectFileTypeFromMagic detects file type from magic bytes

decoder/stream/protobuf/protobuf.go

@@ -37,6 +37,7 @@ import (
 	decoderconfig "github.com/dreadl0ck/netcap/decoder/config"
 	"github.com/dreadl0ck/netcap/decoder/core"
 	streamutils "github.com/dreadl0ck/netcap/decoder/stream/utils"
+	"github.com/dreadl0ck/netcap/internal/entropy"
 	logging "github.com/dreadl0ck/netcap/internal/logger"
 	"github.com/dreadl0ck/netcap/types"
 )
@@ -602,24 +603,7 @@ func IsPrintable(data []byte) bool {
 
 // CalculateEntropy computes Shannon entropy of the data in bits.
 func CalculateEntropy(data []byte) float64 {
-	if len(data) == 0 {
-		return 0
-	}
-
-	freq := make(map[byte]int)
-	for _, b := range data {
-		freq[b]++
-	}
-
-	entropy := 0.0
-	length := float64(len(data))
-
-	for _, count := range freq {
-		p := float64(count) / length
-		entropy -= p * math.Log2(p)
-	}
-
-	return entropy
+	return entropy.Bytes(data)
 }
 
 // DetectMessageType classifies a decoded message based on field patterns.

internal/entropy/entropy.go

@@ -0,0 +1,26 @@
+// Package entropy calculates Shannon entropy over byte distributions.
+package entropy
+
+import "math"
+
+// Bytes returns Shannon entropy in bits per byte, or zero for empty input.
+func Bytes(data []byte) float64 {
+	if len(data) == 0 {
+		return 0
+	}
+
+	var counts [256]int
+	for _, b := range data {
+		counts[b]++
+	}
+
+	var result float64
+	length := float64(len(data))
+	for _, count := range counts {
+		if count > 0 {
+			p := float64(count) / length
+			result -= p * math.Log2(p)
+		}
+	}
+	return result
+}

internal/entropy/entropy_test.go

@@ -0,0 +1,159 @@
+package entropy
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+	"math/rand"
+	"testing"
+)
+
+const entropyTolerance = 1e-12
+
+var entropySink float64
+
+func TestBytes(t *testing.T) {
+	all := make([]byte, 256)
+	for i := range all {
+		all[i] = byte(i)
+	}
+	for _, tc := range []struct {
+		name string
+		data []byte
+		want float64
+	}{
+		{"nil", nil, 0},
+		{"empty", []byte{}, 0},
+		{"singleton", []byte{255}, 0},
+		{"repeated", bytes.Repeat([]byte{42}, 4096), 0},
+		{"balanced", bytes.Repeat([]byte{0, 255}, 128), 1},
+		{"all256", all, 8},
+		{"skewed", []byte{0, 0, 0, 255}, -0.75*math.Log2(0.75) - 0.25*math.Log2(0.25)},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := Bytes(tc.data); math.IsNaN(got) || math.Abs(got-tc.want) > entropyTolerance {
+				t.Fatalf("Bytes = %.17g, want %.17g", got, tc.want)
+			}
+			if allocs := testing.AllocsPerRun(100, func() { entropySink = Bytes(tc.data) }); allocs != 0 {
+				t.Errorf("Bytes allocated %g times, want zero", allocs)
+			}
+		})
+	}
+}
+
+func TestBytesRandomized(t *testing.T) {
+	rng := rand.New(rand.NewSource(1))
+	for i := range 200 {
+		data := make([]byte, rng.Intn(64*1024+1))
+		alphabet := 1 + rng.Intn(256)
+		for j := range data {
+			data[j] = byte(rng.Intn(alphabet))
+		}
+		got, want := Bytes(data), repeatedCountEntropy(data)
+		if math.IsNaN(got) || math.Abs(got-want) > entropyTolerance {
+			t.Fatalf("case %d (size %d, alphabet %d): Bytes = %.17g, want %.17g", i, len(data), alphabet, got, want)
+		}
+	}
+}
+
+func FuzzBytes(f *testing.F) {
+	f.Add([]byte{})
+	f.Add([]byte{255})
+	f.Add(bytes.Repeat([]byte{42}, 512))
+	f.Add([]byte{0, 255, 0, 255})
+	f.Add([]byte{0, 0, 0, 255})
+	all := make([]byte, 256)
+	for i := range all {
+		all[i] = byte(i)
+	}
+	f.Add(all)
+	f.Add(entropyData(1500, "text"))
+	f.Add(entropyData(64*1024, "random"))
+	f.Fuzz(func(t *testing.T, data []byte) {
+		if len(data) > 64*1024 {
+			data = data[:64*1024]
+		}
+		got, want := Bytes(data), repeatedCountEntropy(data)
+		if math.IsNaN(got) || math.Abs(got-want) > entropyTolerance {
+			t.Fatalf("size %d: Bytes = %.17g, want %.17g", len(data), got, want)
+		}
+	})
+}
+
+// repeatedCountEntropy is the old packet implementation and differential reference.
+func repeatedCountEntropy(data []byte) (entropy float64) {
+	if len(data) == 0 {
+		return 0
+	}
+	for i := range 256 {
+		p := float64(bytes.Count(data, []byte{byte(i)})) / float64(len(data))
+		if p > 0 {
+			entropy += -p * math.Log2(p)
+		}
+	}
+	return entropy
+}
+
+func protobufMapEntropy(data []byte) float64 {
+	if len(data) == 0 {
+		return 0
+	}
+	freq := make(map[byte]int)
+	for _, b := range data {
+		freq[b]++
+	}
+	entropy := 0.0
+	length := float64(len(data))
+	for _, count := range freq {
+		p := float64(count) / length
+		entropy -= p * math.Log2(p)
+	}
+	return entropy
+}
+
+func entropyData(size int, distribution string) []byte {
+	data := make([]byte, size)
+	switch distribution {
+	case "repeated":
+		for i := range data {
+			data[i] = 42
+		}
+	case "text":
+		const text = "GET /index.html HTTP/1.1\r\nHost: example.com\r\nContent-Type: text/plain\r\n\r\nThe quick brown fox jumps over the lazy dog.\n"
+		for i := range data {
+			data[i] = text[i%len(text)]
+		}
+	case "random":
+		rng := rand.New(rand.NewSource(1))
+		for i := range data {
+			data[i] = byte(rng.Intn(256))
+		}
+	default:
+		panic("unknown entropy distribution: " + distribution)
+	}
+	return data
+}
+
+func BenchmarkBytes(b *testing.B) {
+	for _, size := range []int{64, 512, 1500, 16384, 1 << 20} {
+		for _, distribution := range []string{"repeated", "text", "random"} {
+			data := entropyData(size, distribution)
+			for _, impl := range []struct {
+				name string
+				fn   func([]byte) float64
+			}{
+				{"Bytes", Bytes},
+				{"OldPacketRepeatedCount", repeatedCountEntropy},
+				{"OldProtobufMap", protobufMapEntropy},
+			} {
+				b.Run(fmt.Sprintf("%d/%s/%s", size, distribution, impl.name), func(b *testing.B) {
+					b.SetBytes(int64(len(data)))
+					b.ReportAllocs()
+					for b.Loop() {
+						entropySink = impl.fn(data)
+					}
+				})
+			}
+		}
+	}
+}

internal/entropy/experiment.go

@@ -0,0 +1,43 @@
+//go:build entropyexperiment
+
+package entropy
+
+import "math"
+
+// Four banks reduce repeated-byte dependencies but add setup and merging work.
+type experimentBanks [4][256]int
+
+func histogramGo4(data []byte, banks *experimentBanks) {
+	for len(data) >= 4 {
+		banks[0][data[0]]++
+		banks[1][data[1]]++
+		banks[2][data[2]]++
+		banks[3][data[3]]++
+		data = data[4:]
+	}
+	for _, v := range data {
+		banks[0][v]++
+	}
+}
+
+func experimentEntropy(banks *experimentBanks, size int) float64 {
+	var result float64
+	for i, count := range banks[0] {
+		count += banks[1][i] + banks[2][i] + banks[3][i]
+		if count > 0 {
+			p := float64(count) / float64(size)
+			result -= p * math.Log2(p)
+		}
+	}
+	return result
+}
+
+// BytesGo4 evaluates the portable four-bank experiment, including setup and logs.
+func BytesGo4(data []byte) float64 {
+	if len(data) == 0 {
+		return 0
+	}
+	var banks experimentBanks
+	histogramGo4(data, &banks)
+	return experimentEntropy(&banks, len(data))
+}

internal/entropy/experiment_arm64.go

@@ -0,0 +1,18 @@
+//go:build entropyexperiment && !purego
+
+package entropy
+
+const experimentASMName = "ASM4"
+
+//go:noescape
+func histogramARM64(data []byte, banks *experimentBanks)
+
+// BytesARM64 evaluates scalar four-bank ARM64 assembly, including setup and logs.
+func BytesARM64(data []byte) float64 {
+	if len(data) == 0 {
+		return 0
+	}
+	var banks experimentBanks
+	histogramARM64(data, &banks)
+	return experimentEntropy(&banks, len(data))
+}