From 5e7c2e9e73743dec5bdd48e08d405c78982843bb Mon Sep 17 00:00:00 2001 From: Ethan Aldrich Date: Fri, 16 Oct 2026 13:27:56 -0700 Subject: [PATCH] Add HCL store for .hcl and .tfvars files Add a store that reads and writes HCL, so Terraform .tfvars and Terragrunt .hcl files can be encrypted directly. The format is selected by the .hcl and .tfvars extensions or with --input-type/--output-type hcl. Attributes and (labeled) blocks are mapped to the sops tree. A block is stored under its header, for example `resource "aws_instance" "web" {`, which keeps it distinct from an attribute holding an object. Literal strings, numbers and bools are encrypted, as are lists and objects made of literals. Any other expression (references, function calls, interpolated templates, ...) is kept as its source text, wrapped as `${...}`, and encrypted like a string, so it is written back verbatim on decryption. Comments are kept; the sops metadata is stored as a top-level `sops` attribute. Fixes #292 Signed-off-by: Ethan Aldrich --- README.rst | 2 +- cmd/sops/common/common.go | 6 + cmd/sops/formats/formats.go | 9 + cmd/sops/formats/formats_test.go | 6 + cmd/sops/main.go | 36 +- config/config.go | 3 + decrypt/decrypt.go | 2 +- functional-tests/res/format.enc.hcl | 23 ++ functional-tests/src/lib.rs | 18 + go.mod | 9 +- go.sum | 16 + stores/hcl/store.go | 536 ++++++++++++++++++++++++++++ stores/hcl/store_test.go | 320 +++++++++++++++++ 13 files changed, 965 insertions(+), 21 deletions(-) create mode 100644 functional-tests/res/format.enc.hcl create mode 100644 stores/hcl/store.go create mode 100644 stores/hcl/store_test.go diff --git a/README.rst b/README.rst index bde72ff8d6..5d2c5f1b6d 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ SOPS: Secrets OPerationS ======================== -**SOPS** is an editor of encrypted files that supports YAML, JSON, ENV, INI and BINARY +**SOPS** is an editor of encrypted files that supports YAML, JSON, ENV, INI, HCL and BINARY formats and encrypts with AWS KMS, GCP KMS, Azure Key Vault, HuaweiCloud KMS, age, and PGP. (`demo `_) diff --git a/cmd/sops/common/common.go b/cmd/sops/common/common.go index 6d6fa0751a..0fa02cd572 100644 --- a/cmd/sops/common/common.go +++ b/cmd/sops/common/common.go @@ -16,6 +16,7 @@ import ( "github.com/getsops/sops/v3/keyservice" "github.com/getsops/sops/v3/kms" "github.com/getsops/sops/v3/stores/dotenv" + "github.com/getsops/sops/v3/stores/hcl" "github.com/getsops/sops/v3/stores/ini" "github.com/getsops/sops/v3/stores/json" "github.com/getsops/sops/v3/stores/yaml" @@ -47,6 +48,10 @@ func newDotenvStore(c *config.StoresConfig) Store { return dotenv.NewStore(&c.Dotenv) } +func newHclStore(c *config.StoresConfig) Store { + return hcl.NewStore(&c.HCL) +} + func newIniStore(c *config.StoresConfig) Store { return ini.NewStore(&c.INI) } @@ -62,6 +67,7 @@ func newYamlStore(c *config.StoresConfig) Store { var storeConstructors = map[Format]storeConstructor{ Binary: newBinaryStore, Dotenv: newDotenvStore, + Hcl: newHclStore, Ini: newIniStore, Json: newJsonStore, Yaml: newYamlStore, diff --git a/cmd/sops/formats/formats.go b/cmd/sops/formats/formats.go index 83c5391e28..cd3a66e6e3 100644 --- a/cmd/sops/formats/formats.go +++ b/cmd/sops/formats/formats.go @@ -8,6 +8,7 @@ type Format int const ( Binary Format = iota Dotenv + Hcl Ini Json Yaml @@ -16,6 +17,7 @@ const ( var stringToFormat = map[string]Format{ "binary": Binary, "dotenv": Dotenv, + "hcl": Hcl, "ini": Ini, "json": Json, "yaml": Yaml, @@ -46,6 +48,11 @@ func IsEnvFile(path string) bool { return strings.HasSuffix(path, ".env") } +// IsHclFile returns true if a given file path corresponds to a HCL file +func IsHclFile(path string) bool { + return strings.HasSuffix(path, ".hcl") || strings.HasSuffix(path, ".tfvars") +} + // IsIniFile returns true if a given file path corresponds to a INI file func IsIniFile(path string) bool { return strings.HasSuffix(path, ".ini") @@ -62,6 +69,8 @@ func FormatForPath(path string) Format { format = Dotenv } else if IsIniFile(path) { format = Ini + } else if IsHclFile(path) { + format = Hcl } return format } diff --git a/cmd/sops/formats/formats_test.go b/cmd/sops/formats/formats_test.go index 7bc6477fae..d247bfb05d 100644 --- a/cmd/sops/formats/formats_test.go +++ b/cmd/sops/formats/formats_test.go @@ -9,6 +9,7 @@ import ( func TestFormatFromString(t *testing.T) { assert.Equal(t, Binary, FormatFromString("foobar")) assert.Equal(t, Dotenv, FormatFromString("dotenv")) + assert.Equal(t, Hcl, FormatFromString("hcl")) assert.Equal(t, Ini, FormatFromString("ini")) assert.Equal(t, Yaml, FormatFromString("yaml")) assert.Equal(t, Json, FormatFromString("json")) @@ -17,6 +18,9 @@ func TestFormatFromString(t *testing.T) { func TestFormatForPath(t *testing.T) { assert.Equal(t, Binary, FormatForPath("/path/to/foobar")) assert.Equal(t, Dotenv, FormatForPath("/path/to/foobar.env")) + assert.Equal(t, Hcl, FormatForPath("/path/to/foobar.hcl")) + assert.Equal(t, Hcl, FormatForPath("/path/to/foobar.tfvars")) + assert.Equal(t, Json, FormatForPath("/path/to/foobar.tfvars.json")) assert.Equal(t, Ini, FormatForPath("/path/to/foobar.ini")) assert.Equal(t, Json, FormatForPath("/path/to/foobar.json")) assert.Equal(t, Yaml, FormatForPath("/path/to/foobar.yml")) @@ -27,6 +31,8 @@ func TestFormatForPathOrString(t *testing.T) { assert.Equal(t, Binary, FormatForPathOrString("/path/to/foobar", "")) assert.Equal(t, Dotenv, FormatForPathOrString("/path/to/foobar", "dotenv")) assert.Equal(t, Dotenv, FormatForPathOrString("/path/to/foobar.env", "")) + assert.Equal(t, Hcl, FormatForPathOrString("/path/to/foobar", "hcl")) + assert.Equal(t, Hcl, FormatForPathOrString("/path/to/foobar.tfvars", "")) assert.Equal(t, Ini, FormatForPathOrString("/path/to/foobar", "ini")) assert.Equal(t, Ini, FormatForPathOrString("/path/to/foobar.ini", "")) assert.Equal(t, Json, FormatForPathOrString("/path/to/foobar", "json")) diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 8092a3f400..0efd069c22 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -307,11 +307,11 @@ func main() { }, cli.StringFlag{ Name: "input-type", - Usage: "currently ini, json, yaml, dotenv and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently ini, json, yaml, dotenv, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently ini, json, yaml, dotenv and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently ini, json, yaml, dotenv, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.StringFlag{ Name: "filename", @@ -517,7 +517,7 @@ func main() { Flags: []cli.Flag{ cli.StringFlag{ Name: "input-type", - Usage: "currently ini, json, yaml, dotenv and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently ini, json, yaml, dotenv, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, }, Action: func(c *cli.Context) error { @@ -732,7 +732,7 @@ func main() { }, cli.StringFlag{ Name: "input-type", - Usage: "currently ini, json, yaml, dotenv and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently ini, json, yaml, dotenv, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, }, keyserviceFlags...), Action: func(c *cli.Context) error { @@ -801,11 +801,11 @@ func main() { }, cli.StringFlag{ Name: "input-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.BoolFlag{ Name: "ignore-mac", @@ -972,11 +972,11 @@ func main() { }, cli.StringFlag{ Name: "input-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.StringFlag{ Name: "unencrypted-suffix", @@ -1125,11 +1125,11 @@ func main() { }, cli.StringFlag{ Name: "input-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.StringFlag{ Name: "encryption-context", @@ -1331,11 +1331,11 @@ func main() { }, cli.StringFlag{ Name: "input-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.StringFlag{ Name: "unencrypted-suffix", @@ -1457,11 +1457,11 @@ func main() { Flags: append([]cli.Flag{ cli.StringFlag{ Name: "input-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.BoolFlag{ Name: "value-file", @@ -1593,11 +1593,11 @@ func main() { Flags: append([]cli.Flag{ cli.StringFlag{ Name: "input-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.IntFlag{ Name: "shamir-secret-sharing-threshold", @@ -1752,11 +1752,11 @@ func main() { }, cli.StringFlag{ Name: "input-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.BoolFlag{ Name: "show-master-keys, s", diff --git a/config/config.go b/config/config.go index 511df1bc15..65a98c8829 100644 --- a/config/config.go +++ b/config/config.go @@ -101,6 +101,8 @@ func FindConfigFile(start string) (string, error) { type DotenvStoreConfig struct{} +type HCLStoreConfig struct{} + type INIStoreConfig struct{} type JSONStoreConfig struct { @@ -117,6 +119,7 @@ type YAMLStoreConfig struct { type StoresConfig struct { Dotenv DotenvStoreConfig `yaml:"dotenv"` + HCL HCLStoreConfig `yaml:"hcl"` INI INIStoreConfig `yaml:"ini"` JSONBinary JSONBinaryStoreConfig `yaml:"json_binary"` JSON JSONStoreConfig `yaml:"json"` diff --git a/decrypt/decrypt.go b/decrypt/decrypt.go index e26cbe4798..98f7623f2b 100644 --- a/decrypt/decrypt.go +++ b/decrypt/decrypt.go @@ -72,7 +72,7 @@ func DataWithFormat(data []byte, format Format) (cleartext []byte, err error) { // Data is a helper that takes encrypted data and a format string, // decrypts the data and returns its cleartext in an []byte. -// The format string can be `json`, `yaml`, `ini`, `dotenv` or `binary`. +// The format string can be `json`, `yaml`, `ini`, `hcl`, `dotenv` or `binary`. // If the format string is empty, binary format is assumed. func Data(data []byte, format string) (cleartext []byte, err error) { formatFmt := FormatFromString(format) diff --git a/functional-tests/res/format.enc.hcl b/functional-tests/res/format.enc.hcl new file mode 100644 index 0000000000..1ca7e91c00 --- /dev/null +++ b/functional-tests/res/format.enc.hcl @@ -0,0 +1,23 @@ +foo = "ENC[AES256_GCM,data:VWTe,iv:1jXojBlsfxaGel2OlCAY9i0CDaaVRCudGjuJlV90EA8=,tag:yzj4lEr4OaccODNka2shBg==,type:str]" + +baz "qux" { + quux = [ + "ENC[AES256_GCM,data:fw==,iv:Qykwql3Kahp2uoSgTLUv60HGniPU839v2i2zoDPuxJ4=,tag:ZYU3fHRnO+spNg0+qJAH/A==,type:int]", + "ENC[AES256_GCM,data:MA==,iv:CDH2zKXaYMhJr4c/528Z/hFM2IfLxlvQSc1cgVH+WT4=,tag:HDpCS1qqOJDU64PCpUcVZQ==,type:int]", + ] + ref = "ENC[AES256_GCM,data:AVv1laLwAWU=,iv:E37lz0Sw+aFKC6sS5xtMELeu4a3DliYQR5z4Qs2Dox0=,tag:x/PC6Lfj7OpNdsployZLhg==,type:str]" +} + +sops = { + lastmodified = "2026-10-16T20:25:39Z" + mac = "ENC[AES256_GCM,data:tg9KJb7A0nHIb+SmlZiry1DDQop8JIiJtN405BLNs1oBD1y3zb1XJjve5k837K6Hm7xTVrb4gw/MgW5c32IyZdjTqEQv+FusdYZ3ElK3OcIFFUoNvbakOBnGBTNA+h28ViH5c6LaH8awVftfdguOxouODqMhvlD5a0mHLy38oAc=,iv:lct8P+0EXKLNg6dh6elZk4kM09dzaljC1fbnCc3vv24=,tag:E8Wu1fqMUSPiuQG4Yg1a4Q==,type:str]" + pgp = [ + { + created_at = "2026-10-16T20:25:39Z" + enc = "-----BEGIN PGP MESSAGE-----\n\nhQEMAyUpShfNkFB/AQf+MBQLTQghvVfi1n2Bu8F+0OP8WW7vx0ZKSUY19piKrh+G\nq8kfvpGkA+xHiQ3DGR034jeTwAcOBPnncJLF+omeeIUEO7frT9I4e7EMMBAKfubF\n5V/me5Oq85OhjvokzlxguTJeZvzgLll7KL4/vletIpUc2SHK9beK5dNiuVVSLjNc\nY4w9zM1t7SGQL/ueRmxCQBywAJsQbA8sclQHYI/0+C/fmjm1qEcYKdLwYDVCXSKB\nDOFhL4acuKu+eMljpxMe4vJ+dp+CQqF59WOKjYicDDSplYc2LVw9i7addD6NRRJJ\nQY7mfBFbATBfDyAAQdRINrbWpS05o/DdE41IQ/hl/tJeAbTDlkI0MI4U8p+LrERa\n6+VyeDX7tC/MNKzSznutBAbYnVRz4sZ4G5EBPJBVpSCcPEa8hmB+zcMaehjGwnFL\ndPXDtRMR4nUgzHi9Iha4OJFo5D+N5F2pFWFyqw1hyg==\n=0/eM\n-----END PGP MESSAGE-----" + fp = "FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4" + }, + ] + unencrypted_suffix = "_unencrypted" + version = "3.13.3" +} diff --git a/functional-tests/src/lib.rs b/functional-tests/src/lib.rs index 698d310516..3f0d243755 100644 --- a/functional-tests/src/lib.rs +++ b/functional-tests/src/lib.rs @@ -1737,6 +1737,24 @@ bar: |- String::from_utf8_lossy(&output.stdout) == "[foo]\nbar = baz\n", "Unexpected decrypted content" ); + + // HCL + let output = Command::new(SOPS_BINARY_PATH) + .arg("decrypt") + .arg("res/format.enc.hcl") + .output() + .expect("Error running sops"); + println!( + "stdout: {}, stderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.status.success(), "SOPS didn't return successfully"); + assert!( + String::from_utf8_lossy(&output.stdout) + == "foo = \"bar\"\n\nbaz \"qux\" {\n quux = [\n 1,\n 2,\n ]\n ref = var.x\n}\n", + "Unexpected decrypted content" + ); } #[test] diff --git a/go.mod b/go.mod index 811f12b2cd..35b416d1a5 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/goware/prefixer v0.0.0-20160118172347-395022866408 github.com/hashicorp/go-cleanhttp v0.5.2 + github.com/hashicorp/hcl/v2 v2.25.0 github.com/hashicorp/vault/api v1.23.0 github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.213 github.com/lib/pq v1.12.3 @@ -35,12 +36,14 @@ require ( github.com/sirupsen/logrus v1.10.2 github.com/stretchr/testify v1.12.1 github.com/urfave/cli v1.22.17 + github.com/zclconf/go-cty v1.19.0 go.yaml.in/yaml/v3 v3.0.5 golang.org/x/crypto v0.55.0 golang.org/x/net v0.58.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sys v0.47.0 golang.org/x/term v0.45.0 + golang.org/x/text v0.41.0 google.golang.org/api v0.294.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 google.golang.org/grpc v1.83.2 @@ -69,6 +72,9 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect + github.com/agext/levenshtein v1.2.1 // indirect + github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/apparentlymart/go-textseg/v17 v17.0.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.1 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1 // indirect @@ -148,9 +154,10 @@ require ( go.opentelemetry.io/otel/sdk v1.45.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect go.opentelemetry.io/otel/trace v1.45.0 // indirect + golang.org/x/mod v0.38.0 // indirect golang.org/x/sync v0.22.0 // indirect - golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect + golang.org/x/tools v0.48.0 // indirect google.golang.org/genproto v0.0.0-20260720171339-e059f2f05d78 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260720171339-e059f2f05d78 // indirect ) diff --git a/go.sum b/go.sum index 24b7aeb953..ade9ffe794 100644 --- a/go.sum +++ b/go.sum @@ -67,6 +67,12 @@ github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEV github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= +github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= +github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= +github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/apparentlymart/go-textseg/v17 v17.0.1 h1:bpMXRgQ5cEoRNuQke1a80/Nl6w3G5eoIbWo9f3gXkAs= +github.com/apparentlymart/go-textseg/v17 v17.0.1/go.mod h1:fa8X4jgGeevslICIY6LcdjkSecWnXmYd9Lk34z/VxZs= github.com/aws/aws-sdk-go-v2 v1.45.1 h1:iIoG3NaLhV6UZpPXyPXlDj2I9oS8tV/nMcMnITCC6Ks= github.com/aws/aws-sdk-go-v2 v1.45.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= @@ -239,6 +245,8 @@ github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9 github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= +github.com/hashicorp/hcl/v2 v2.25.0 h1:HmmQVYRny4MaBo4b20TjmL46wyuUxpnMWkPZ4+NTbWk= +github.com/hashicorp/hcl/v2 v2.25.0/go.mod h1:vR+FKETxoZAmRlHgFfKmuqivj+C4Izm/c66XkmZ3r7M= github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A= github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko= github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.213 h1:uKt4C7kPCCDk47+VLTPLUY/u4si2wr+SLFI7EEM8KSc= @@ -336,6 +344,10 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zclconf/go-cty v1.19.0 h1:IV8WdqYZc2c5rLX9bEoLNXKojBAp0MZPBHMIrCoa/s4= +github.com/zclconf/go-cty v1.19.0/go.mod h1:12W89jGn3JCOIQi7infWr9m80rOkb5RNYJqXMZcN4c8= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo= go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU= go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ= @@ -375,6 +387,8 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -434,6 +448,8 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/stores/hcl/store.go b/stores/hcl/store.go new file mode 100644 index 0000000000..7e63ba1c71 --- /dev/null +++ b/stores/hcl/store.go @@ -0,0 +1,536 @@ +/* +Package hcl implements a SOPS store for HCL (HashiCorp Configuration Language) +files such as Terraform `.tfvars` and Terragrunt `.hcl` files. + +Mapping between HCL and the SOPS tree: + + - An attribute becomes a tree item whose key is the attribute name. + - A block becomes a tree item whose key is the block header as it is written + in the file, including labels and the opening brace, for example + `resource "aws_instance" "web" {`. The trailing brace is what tells blocks + apart from attributes holding an object. The value is the block body. + - Literal strings, numbers and booleans become strings, ints/floats and bools, + and are encrypted. `null` becomes nil. + - Tuples and objects made only of literals become lists and mappings. + - Every other expression (references, function calls, conditionals, + templates with interpolation, ...) is opaque: its source text is wrapped as + `${}` and treated as a string. It is therefore encrypted like any + other string, and written back verbatim on decryption. This is the same + convention HCL's JSON syntax uses for expressions. + - Comments become sops.Comment items. +*/ +package hcl //import "github.com/getsops/sops/v3/stores/hcl" + +import ( + "bytes" + "encoding/json" + "fmt" + "math/big" + "sort" + "strconv" + "strings" + + "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" + "github.com/getsops/sops/v3/stores" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/zclconf/go-cty/cty" + "golang.org/x/text/unicode/norm" +) + +const indent = " " + +// Store handles storage of HCL data. +type Store struct { + config *config.HCLStoreConfig +} + +// NewStore creates a new HCL store. +func NewStore(c *config.HCLStoreConfig) *Store { + return &Store{config: c} +} + +// Name returns the name of the store. +func (store *Store) Name() string { + return "hcl" +} + +// isExpr reports whether a tree string stands for an opaque expression. +func isExpr(s string) bool { + return len(s) >= 3 && strings.HasPrefix(s, "${") && strings.HasSuffix(s, "}") +} + +func quote(s string) string { + return string(hclwrite.TokensForValue(cty.StringVal(s)).Bytes()) +} + +// --------------------------------------------------------------------------- +// HCL -> tree + +type span struct { + start, end int +} + +type comment struct { + span + text string +} + +type loader struct { + src []byte + comments []comment +} + +func (l *loader) hasComment(s span) bool { + for _, c := range l.comments { + if c.start >= s.start && c.end <= s.end { + return true + } + } + return false +} + +func commentText(raw string) string { + switch { + case strings.HasPrefix(raw, "/*"): + raw = strings.TrimSuffix(strings.TrimPrefix(raw, "/*"), "*/") + return strings.TrimSpace(raw) + case strings.HasPrefix(raw, "//"): + raw = raw[2:] + default: + raw = strings.TrimPrefix(raw, "#") + } + return strings.TrimPrefix(strings.TrimRight(raw, "\r\n"), " ") +} + +func (l *loader) opaque(e hclsyntax.Expression) string { + r := e.Range() + return "${" + string(l.src[r.Start.Byte:r.End.Byte]) + "}" +} + +func (l *loader) literal(s string) (interface{}, error) { + if isExpr(s) { + return nil, fmt.Errorf("string %q starts with ${ and ends with }, which cannot be told apart from an expression", s) + } + return s, nil +} + +func number(v cty.Value) (interface{}, bool) { + bf := v.AsBigFloat() + if bf.IsInt() { + if i, acc := bf.Int64(); acc == big.Exact && int64(int(i)) == i { + return int(i), true + } + } + f, _ := bf.Float64() + if r, err := cty.ParseNumberVal(strconv.FormatFloat(f, 'f', -1, 64)); err == nil && r.RawEquals(v) { + return f, true + } + return nil, false +} + +// nfc reports whether the source of e is NFC-normal. HCL normalizes decoded +// strings to NFC, which would change the bytes of anything that is not. +func (l *loader) nfc(r hcl.Range) bool { + src := l.src[r.Start.Byte:r.End.Byte] + // \u and \U escapes are ASCII in the source but can decode to non-NFC text. + return norm.NFC.IsNormal(src) && !bytes.Contains(src, []byte(`\u`)) && !bytes.Contains(src, []byte(`\U`)) +} + +func (l *loader) value(e hclsyntax.Expression) (interface{}, error) { + r := e.Range() + rs := span{r.Start.Byte, r.End.Byte} + switch e := e.(type) { + case *hclsyntax.UnaryOpExpr: + if lv, ok := e.Val.(*hclsyntax.LiteralValueExpr); ok && e.Op == hclsyntax.OpNegate && lv.Val.Type() == cty.Number { + if n, ok := number(lv.Val.Negate()); ok { + return n, nil + } + } + case *hclsyntax.LiteralValueExpr: + switch { + case e.Val.IsNull(): + return nil, nil + case e.Val.Type() == cty.Bool: + return e.Val.True(), nil + case e.Val.Type() == cty.Number: + if n, ok := number(e.Val); ok { + return n, nil + } + } + case *hclsyntax.TemplateExpr: + var sb strings.Builder + literal := true + for _, p := range e.Parts { + lv, ok := p.(*hclsyntax.LiteralValueExpr) + if !ok || lv.Val.Type() != cty.String { + literal = false + break + } + sb.WriteString(lv.Val.AsString()) + } + if literal && l.nfc(r) { + return l.literal(sb.String()) + } + case *hclsyntax.TupleConsExpr: + if l.hasComment(rs) { + break + } + out := make([]interface{}, 0, len(e.Exprs)) + for _, item := range e.Exprs { + v, err := l.value(item) + if err != nil { + return nil, err + } + out = append(out, v) + } + return out, nil + case *hclsyntax.ObjectConsExpr: + if l.hasComment(rs) { + break + } + out := sops.TreeBranch{} + ok := true + for _, item := range e.Items { + key := l.objectKey(item.KeyExpr) + if key == nil { + ok = false + break + } + v, err := l.value(item.ValueExpr) + if err != nil { + return nil, err + } + out = append(out, sops.TreeItem{Key: *key, Value: v}) + } + if ok { + return out, nil + } + } + return l.opaque(e), nil +} + +// objectKey returns the key of an object entry if it is a plain identifier or +// a literal string, and nil otherwise. +func (l *loader) objectKey(e hclsyntax.Expression) *string { + if kw := hcl.ExprAsKeyword(e); kw != "" { + return &kw + } + if w, ok := e.(*hclsyntax.ObjectConsKeyExpr); ok { + e = w.Wrapped + } + if t, ok := e.(*hclsyntax.TemplateExpr); ok && len(t.Parts) == 1 { + if lv, ok := t.Parts[0].(*hclsyntax.LiteralValueExpr); ok && lv.Val.Type() == cty.String { + if !l.nfc(t.Range()) { + return nil + } + s := lv.Val.AsString() + return &s + } + } + return nil +} + +type bodyItem struct { + span + attr *hclsyntax.Attribute + block *hclsyntax.Block +} + +// body converts the contents of body, which spans the bytes [lo, hi) of the +// source, to a tree branch. +func (l *loader) body(b *hclsyntax.Body, lo, hi int) (sops.TreeBranch, error) { + var items []bodyItem + for _, a := range b.Attributes { + items = append(items, bodyItem{span{a.SrcRange.Start.Byte, a.SrcRange.End.Byte}, a, nil}) + } + for _, blk := range b.Blocks { + items = append(items, bodyItem{span{blk.TypeRange.Start.Byte, blk.CloseBraceRange.End.Byte}, nil, blk}) + } + sort.Slice(items, func(i, j int) bool { return items[i].start < items[j].start }) + + var branch sops.TreeBranch + // addComments emits the comments located in [from, to). + addComments := func(from, to int, prev *bodyItem) { + for _, c := range l.comments { + if c.start < from || c.end > to { + continue + } + inline := prev != nil && prev.attr != nil && + !strings.Contains(string(l.src[prev.end:c.start]), "\n") + branch = append(branch, sops.TreeItem{Key: sops.Comment{Value: c.text, Inline: inline}}) + } + } + pos := lo + var prev *bodyItem + for i := range items { + it := &items[i] + addComments(pos, it.start, prev) + var ti sops.TreeItem + if it.attr != nil { + v, err := l.value(it.attr.Expr) + if err != nil { + return nil, fmt.Errorf("attribute %q: %w", it.attr.Name, err) + } + ti = sops.TreeItem{Key: it.attr.Name, Value: v} + } else { + blk := it.block + header := blk.Type + for i, label := range blk.Labels { + if !l.nfc(blk.LabelRanges[i]) { + return nil, fmt.Errorf("block label %q is not NFC-normalized, which HCL cannot represent", label) + } + header += " " + quote(label) + } + inner, err := l.body(blk.Body, blk.OpenBraceRange.End.Byte, blk.CloseBraceRange.Start.Byte) + if err != nil { + return nil, err + } + ti = sops.TreeItem{Key: header + " {", Value: inner} + } + branch = append(branch, ti) + pos = it.end + prev = it + } + addComments(pos, hi, prev) + return branch, nil +} + +// LoadPlainFile loads a plaintext HCL file's bytes onto a sops.TreeBranches runtime object +func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { + file, diags := hclsyntax.ParseConfig(in, "", hcl.InitialPos) + if diags.HasErrors() { + return nil, fmt.Errorf("Could not unmarshal input data: %s", diags.Error()) + } + tokens, diags := hclsyntax.LexConfig(in, "", hcl.InitialPos) + if diags.HasErrors() { + return nil, fmt.Errorf("Could not unmarshal input data: %s", diags.Error()) + } + l := &loader{src: in} + for _, t := range tokens { + if t.Type == hclsyntax.TokenComment { + l.comments = append(l.comments, comment{span{t.Range.Start.Byte, t.Range.End.Byte}, commentText(string(t.Bytes))}) + } + } + branch, err := l.body(file.Body.(*hclsyntax.Body), 0, len(in)) + if err != nil { + return nil, fmt.Errorf("Could not unmarshal input data: %s", err) + } + return sops.TreeBranches{branch}, nil +} + +// LoadEncryptedFile loads encrypted HCL file's bytes onto a sops.Tree runtime object +func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) { + branches, err := store.LoadPlainFile(in) + if err != nil { + return sops.Tree{}, err + } + branches, metadata, err := stores.ExtractMetadata(branches, stores.MetadataOpts{}) + if err != nil { + return sops.Tree{}, err + } + return sops.Tree{ + Branches: branches, + Metadata: metadata, + }, nil +} + +// --------------------------------------------------------------------------- +// tree -> HCL + +func isIdent(s string) bool { + return hclsyntax.ValidIdentifier(s) +} + +func scalar(v interface{}) (string, error) { + switch v := v.(type) { + case nil: + return "null", nil + case string: + if isExpr(v) { + return v[2 : len(v)-1], nil + } + return quote(v), nil + case []byte: + return quote(string(v)), nil + case bool: + return strconv.FormatBool(v), nil + case int: + return strconv.Itoa(v), nil + case float64: + if v != v || v > 1.7976931348623157e308 || v < -1.7976931348623157e308 { + return "", fmt.Errorf("cannot represent %v in HCL", v) + } + return strconv.FormatFloat(v, 'f', -1, 64), nil + } + return "", fmt.Errorf("unsupported value type %T", v) +} + +func writeComments(sb *strings.Builder, c sops.Comment, pad string) { + for _, line := range strings.Split(c.Value, "\n") { + sb.WriteString(pad + strings.TrimRight("# "+line, " ") + "\n") + } +} + +func writeValue(sb *strings.Builder, v interface{}, pad string) error { + switch v := v.(type) { + case sops.TreeBranch: + if len(v) == 0 { + sb.WriteString("{}") + return nil + } + sb.WriteString("{\n") + if err := writeItems(sb, v, pad+indent, false); err != nil { + return err + } + sb.WriteString(pad + "}") + case []interface{}: + if len(v) == 0 { + sb.WriteString("[]") + return nil + } + sb.WriteString("[\n") + for _, e := range v { + if c, ok := e.(sops.Comment); ok { + writeComments(sb, c, pad+indent) + continue + } + sb.WriteString(pad + indent) + if err := writeValue(sb, e, pad+indent); err != nil { + return err + } + sb.WriteString(",\n") + } + sb.WriteString(pad + "]") + default: + s, err := scalar(v) + if err != nil { + return err + } + sb.WriteString(s) + } + return nil +} + +func isBlock(key string) bool { + return strings.HasSuffix(key, "{") +} + +// writeItems writes the items of a branch. Blocks are only allowed if blocks is true. +func writeItems(sb *strings.Builder, branch sops.TreeBranch, pad string, blocks bool) error { + var pending []sops.Comment + prevBlock, first := false, true + // open reports that the last attribute line is still waiting for its newline, + // so that an inline comment can be appended to it. + open := false + closeLine := func() { + if open { + sb.WriteString("\n") + open = false + } + } + flush := func() { + for _, c := range pending { + writeComments(sb, c, pad) + } + pending = nil + } + for _, item := range branch { + if c, ok := item.Key.(sops.Comment); ok { + if c.Inline && open { + sb.WriteString(" # " + strings.ReplaceAll(c.Value, "\n", " ")) + closeLine() + } else { + closeLine() + pending = append(pending, c) + } + continue + } + closeLine() + key, ok := item.Key.(string) + if !ok { + return fmt.Errorf("unexpected key type %T", item.Key) + } + block := blocks && isBlock(key) + if !first && (block || prevBlock) { + sb.WriteString("\n") + } + first, prevBlock = false, block + flush() + if block { + sb.WriteString(pad + key + "\n") + inner, ok := item.Value.(sops.TreeBranch) + if !ok { + return fmt.Errorf("block %q: value must be a mapping", key) + } + if err := writeItems(sb, inner, pad+indent, true); err != nil { + return err + } + sb.WriteString(pad + "}\n") + continue + } + if !isIdent(key) { + key = quote(key) + } + sb.WriteString(pad + key + " = ") + if err := writeValue(sb, item.Value, pad); err != nil { + return fmt.Errorf("%s: %w", key, err) + } + open = true + } + closeLine() + flush() + return nil +} + +// EmitPlainFile returns the plaintext HCL file bytes corresponding to a sops.TreeBranches object +func (store *Store) EmitPlainFile(in sops.TreeBranches) ([]byte, error) { + var sb strings.Builder + for _, branch := range in { + if err := writeItems(&sb, branch, "", true); err != nil { + return nil, fmt.Errorf("Error marshaling to HCL: %s", err) + } + } + out := hclwrite.Format([]byte(sb.String())) + // Guard against values that would produce a syntactically invalid file. + if _, diags := hclsyntax.ParseConfig(out, "", hcl.InitialPos); diags.HasErrors() { + return nil, fmt.Errorf("Error marshaling to HCL: %s", diags.Error()) + } + return out, nil +} + +// EmitEncryptedFile returns encrypted HCL file bytes corresponding to a sops.Tree +// runtime object +func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { + branches, err := stores.SerializeMetadata(in, stores.MetadataOpts{}) + if err != nil { + return nil, fmt.Errorf("Error marshaling metadata: %s", err) + } + return store.EmitPlainFile(branches) +} + +// EmitValue returns a single value encoded in a generic interface{} as bytes +func (store *Store) EmitValue(v interface{}) ([]byte, error) { + switch v := v.(type) { + case sops.TreeBranches: + return store.EmitPlainFile(v) + default: + return json.Marshal(v) + } +} + +// EmitExample returns the plaintext HCL file bytes corresponding to the FlatTree example +func (store *Store) EmitExample() []byte { + bytes, err := store.EmitPlainFile(stores.ExampleFlatTree.Branches) + if err != nil { + panic(err) + } + return bytes +} + +// HasSopsTopLevelKey checks whether a top-level "sops" key exists. +func (store *Store) HasSopsTopLevelKey(branch sops.TreeBranch) bool { + return stores.HasSopsTopLevelKey(branch) +} diff --git a/stores/hcl/store_test.go b/stores/hcl/store_test.go new file mode 100644 index 0000000000..bf92878b5d --- /dev/null +++ b/stores/hcl/store_test.go @@ -0,0 +1,320 @@ +package hcl + +import ( + "strings" + "testing" + "time" + + "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/aes" + "github.com/getsops/sops/v3/age" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Inputs are in the format emitted by the store, so they must round-trip unchanged. +var roundTripCases = map[string]string{ + "attributes": `region = "eu-west-1" +count = 3 +ratio = 0.5 +enabled = true +disabled = false +nothing = null +`, + "lists and maps": `ports = [ + 80, + 443, +] +tags = { + env = "prod" + "with space" = "yes" + nested = { + a = 1 + } +} +empty_list = [] +empty_map = {} +mixed = [ + { + a = 1 + }, + [ + true, + ], +] +`, + "blocks": `variable "name" { + default = "x" +} + +resource "aws_instance" "web" { + ami = "ami-123" + + ebs_block_device { + size = 8 + } + + ebs_block_device { + size = 16 + } +} + +empty { +} +`, + "opaque expressions": `a = var.password +b = "prefix-${var.name}" +c = file("${path.module}/key.pem") +d = var.enabled ? "x" : "y" +e = lookup(var.m, "k", null) +f = [ + var.a, + "literal", +] +g = { + k = local.v +} +h = merge( + var.a, + { + k = "v" + }, +) +`, + "comments": `# leading +a = 1 # inline +# between +b = 2 + +# before block +block "x" { + # first in block + c = 3 # inline in block + # last in block +} +# trailing +`, + "escapes": `s = "line1\nline2 \"quoted\" $${not_interp} %%{not_directive}" +`, +} + +func TestRoundTrip(t *testing.T) { + for name, in := range roundTripCases { + t.Run(name, func(t *testing.T) { + store := &Store{} + branches, err := store.LoadPlainFile([]byte(in)) + require.NoError(t, err) + out, err := store.EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, in, string(out)) + }) + } +} + +func TestTreeShape(t *testing.T) { + in := `# top +password = "hunter2" # why +port = 8080 +ratio = 0.1 +ref = var.x +obj = { a = "b" } + +provider "aws" { + region = "us-east-1" +} +block { +} +` + branches, err := (&Store{}).LoadPlainFile([]byte(in)) + require.NoError(t, err) + assert.Equal(t, sops.TreeBranches{{ + {Key: sops.Comment{Value: "top"}}, + {Key: "password", Value: "hunter2"}, + {Key: sops.Comment{Value: "why", Inline: true}}, + {Key: "port", Value: 8080}, + {Key: "ratio", Value: 0.1}, + {Key: "ref", Value: "${var.x}"}, + {Key: "obj", Value: sops.TreeBranch{{Key: "a", Value: "b"}}}, + {Key: `provider "aws" {`, Value: sops.TreeBranch{{Key: "region", Value: "us-east-1"}}}, + {Key: "block {", Value: sops.TreeBranch(nil)}, + }}, branches) +} + +func TestBlockAndObjectAreDistinct(t *testing.T) { + in := "a {\n x = 1\n}\n\nb = {\n x = 1\n}\n" + store := &Store{} + branches, err := store.LoadPlainFile([]byte(in)) + require.NoError(t, err) + out, err := store.EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, in, string(out)) +} + +func TestCommentInsideLiteralStaysOpaque(t *testing.T) { + in := "l = [\n \"a\", # keep me\n \"b\",\n]\n" + store := &Store{} + branches, err := store.LoadPlainFile([]byte(in)) + require.NoError(t, err) + assert.Equal(t, "${[\n \"a\", # keep me\n \"b\",\n]}", branches[0][0].Value) + out, err := store.EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, in, string(out)) +} + +func TestBlockCommentsAreNormalized(t *testing.T) { + branches, err := (&Store{}).LoadPlainFile([]byte("/* one\ntwo */\n// three\na = 1\n")) + require.NoError(t, err) + out, err := (&Store{}).EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, "# one\n# two\n# three\na = 1\n", string(out)) +} + +func TestLiteralLookingLikeExpressionIsRejected(t *testing.T) { + _, err := (&Store{}).LoadPlainFile([]byte(`a = "$${x}"`)) + assert.ErrorContains(t, err, "cannot be told apart") +} + +func TestInvalidInput(t *testing.T) { + _, err := (&Store{}).LoadPlainFile([]byte(`a = = 1`)) + assert.Error(t, err) +} + +func TestEmitRejectsInvalidExpression(t *testing.T) { + _, err := (&Store{}).EmitPlainFile(sops.TreeBranches{{{Key: "a", Value: "${1 +}"}}}) + assert.Error(t, err) +} + +func TestEmitFromOtherFormats(t *testing.T) { + out, err := (&Store{}).EmitPlainFile(sops.TreeBranches{{ + {Key: "m", Value: sops.TreeBranch{{Key: "not an ident", Value: 1}}}, + {Key: "list", Value: []interface{}{"a", sops.TreeBranch{{Key: "k", Value: 2.5}}}}, + }}) + require.NoError(t, err) + assert.Equal(t, "m = {\n \"not an ident\" = 1\n}\nlist = [\n \"a\",\n {\n k = 2.5\n },\n]\n", string(out)) +} + +func TestNoMetadata(t *testing.T) { + _, err := (&Store{}).LoadEncryptedFile([]byte(`hello = 2`)) + assert.Equal(t, sops.MetadataNotFound, err) +} + +func TestHasSopsTopLevelKey(t *testing.T) { + store := &Store{} + assert.False(t, store.HasSopsTopLevelKey(sops.TreeBranch{{Key: "a", Value: "b"}})) + assert.True(t, store.HasSopsTopLevelKey(sops.TreeBranch{{Key: "sops", Value: "b"}})) +} + +// TestEncryptDecrypt runs a file through the real cipher: every literal and every +// opaque expression must be encrypted, metadata must survive the file, and +// decrypting must give back the input. +func TestEncryptDecrypt(t *testing.T) { + in := roundTripCases["opaque expressions"] + + "# leading\nport = 8080\nratio = 0.1\npassword = \"hunter2\"\nplain_unencrypted = \"visible\"\n" + + "resource \"a\" \"b\" {\n key = \"secret\"\n}\n" + in = string(mustFormat(t, in)) + store := &Store{} + branches, err := store.LoadPlainFile([]byte(in)) + require.NoError(t, err) + + dataKey := []byte("0123456789abcdef0123456789abcdef") + tree := sops.Tree{ + Branches: branches, + Metadata: sops.Metadata{ + UnencryptedSuffix: "_unencrypted", + LastModified: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + Version: "3.0.0", + KeyGroups: []sops.KeyGroup{{&age.MasterKey{ + Recipient: "age1yt3tfqlfrwdwx0z0ynwl3jvqmc5eg2zn2c4w5zcdlp0r92x2cwuq9n0qs8", + EncryptedKey: "-----BEGIN AGE ENCRYPTED FILE-----\nabc\n-----END AGE ENCRYPTED FILE-----\n", + }}}, + }, + } + cipher := aes.NewCipher() + mac, err := tree.Encrypt(dataKey, cipher) + require.NoError(t, err) + tree.Metadata.MessageAuthenticationCode, err = cipher.Encrypt(mac, dataKey, tree.Metadata.LastModified.Format(time.RFC3339)) + require.NoError(t, err) + + enc, err := store.EmitEncryptedFile(tree) + require.NoError(t, err) + for _, secret := range []string{"hunter2", "secret", "var.password", "var.name", "0.1", "leading"} { + assert.NotContains(t, strings.ReplaceAll(string(enc), "ENC[", ""), secret) + } + assert.Contains(t, string(enc), `plain_unencrypted = "visible"`) + assert.Contains(t, string(enc), `resource "a" "b" {`) + + loaded, err := store.LoadEncryptedFile(enc) + require.NoError(t, err) + assert.Equal(t, tree.Metadata.LastModified, loaded.Metadata.LastModified) + assert.Equal(t, tree.Metadata.UnencryptedSuffix, loaded.Metadata.UnencryptedSuffix) + require.Len(t, loaded.Metadata.KeyGroups, 1) + key := loaded.Metadata.KeyGroups[0][0].(*age.MasterKey) + assert.Equal(t, "-----BEGIN AGE ENCRYPTED FILE-----\nabc\n-----END AGE ENCRYPTED FILE-----\n", key.EncryptedKey) + + computed, err := loaded.Decrypt(dataKey, cipher) + require.NoError(t, err) + assert.Equal(t, mac, computed) + out, err := store.EmitPlainFile(loaded.Branches) + require.NoError(t, err) + assert.Equal(t, in, string(out)) +} + +func mustFormat(t *testing.T, in string) []byte { + branches, err := (&Store{}).LoadPlainFile([]byte(in)) + require.NoError(t, err) + out, err := (&Store{}).EmitPlainFile(branches) + require.NoError(t, err) + return out +} + +func TestEmitExample(t *testing.T) { + assert.NotEmpty(t, (&Store{}).EmitExample()) +} + +func TestNegativeNumbersKeepTheirType(t *testing.T) { + branches, err := (&Store{}).LoadPlainFile([]byte("i = -5\nf = -0.5\nx = -var.a\n")) + require.NoError(t, err) + assert.Equal(t, sops.TreeBranch{ + {Key: "i", Value: -5}, + {Key: "f", Value: -0.5}, + {Key: "x", Value: "${-var.a}"}, + }, branches[0]) + out, err := (&Store{}).EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, "i = -5\nf = -0.5\nx = -var.a\n", string(out)) +} + +func TestObjectKeyLookingLikeBlock(t *testing.T) { + in := "a = {\n \"foo {\" = \"x\"\n}\n" + store := &Store{} + branches, err := store.LoadPlainFile([]byte(in)) + require.NoError(t, err) + out, err := store.EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, in, string(out)) +} + +// HCL normalizes decoded strings to NFC; anything that is not NFC must be kept as source. +func TestNonNFCStringsAreKeptByteExact(t *testing.T) { + in := "s = \"café\"\nl = [\n \"café\",\n]\nm = {\n \"café\" = 1\n}\nn = \"café\"\n" + store := &Store{} + branches, err := store.LoadPlainFile([]byte(in)) + require.NoError(t, err) + assert.Equal(t, "${\"café\"}", branches[0][0].Value) + assert.Equal(t, "café", branches[0][3].Value) + out, err := store.EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, []byte(in), out) + + // The same text written with ASCII escapes must not be decoded either. + esc := "esc = \"e\\u0301\"\nobj = {\n \"e\\U00000301\" = 1\n}\n" + branches, err = store.LoadPlainFile([]byte(esc)) + require.NoError(t, err) + assert.Equal(t, "${\"e\\u0301\"}", branches[0][0].Value) + out, err = store.EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, esc, string(out)) + + _, err = store.LoadPlainFile([]byte("b \"café\" {\n}\n")) + assert.ErrorContains(t, err, "NFC") +}