diff --git a/README.rst b/README.rst index bde72ff8d6..5d2c5f1b6d 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ SOPS: Secrets OPerationS ======================== -**SOPS** is an editor of encrypted files that supports YAML, JSON, ENV, INI and BINARY +**SOPS** is an editor of encrypted files that supports YAML, JSON, ENV, INI, HCL and BINARY formats and encrypts with AWS KMS, GCP KMS, Azure Key Vault, HuaweiCloud KMS, age, and PGP. (`demo `_) diff --git a/cmd/sops/common/common.go b/cmd/sops/common/common.go index 6d6fa0751a..0fa02cd572 100644 --- a/cmd/sops/common/common.go +++ b/cmd/sops/common/common.go @@ -16,6 +16,7 @@ import ( "github.com/getsops/sops/v3/keyservice" "github.com/getsops/sops/v3/kms" "github.com/getsops/sops/v3/stores/dotenv" + "github.com/getsops/sops/v3/stores/hcl" "github.com/getsops/sops/v3/stores/ini" "github.com/getsops/sops/v3/stores/json" "github.com/getsops/sops/v3/stores/yaml" @@ -47,6 +48,10 @@ func newDotenvStore(c *config.StoresConfig) Store { return dotenv.NewStore(&c.Dotenv) } +func newHclStore(c *config.StoresConfig) Store { + return hcl.NewStore(&c.HCL) +} + func newIniStore(c *config.StoresConfig) Store { return ini.NewStore(&c.INI) } @@ -62,6 +67,7 @@ func newYamlStore(c *config.StoresConfig) Store { var storeConstructors = map[Format]storeConstructor{ Binary: newBinaryStore, Dotenv: newDotenvStore, + Hcl: newHclStore, Ini: newIniStore, Json: newJsonStore, Yaml: newYamlStore, diff --git a/cmd/sops/formats/formats.go b/cmd/sops/formats/formats.go index 83c5391e28..cd3a66e6e3 100644 --- a/cmd/sops/formats/formats.go +++ b/cmd/sops/formats/formats.go @@ -8,6 +8,7 @@ type Format int const ( Binary Format = iota Dotenv + Hcl Ini Json Yaml @@ -16,6 +17,7 @@ const ( var stringToFormat = map[string]Format{ "binary": Binary, "dotenv": Dotenv, + "hcl": Hcl, "ini": Ini, "json": Json, "yaml": Yaml, @@ -46,6 +48,11 @@ func IsEnvFile(path string) bool { return strings.HasSuffix(path, ".env") } +// IsHclFile returns true if a given file path corresponds to a HCL file +func IsHclFile(path string) bool { + return strings.HasSuffix(path, ".hcl") || strings.HasSuffix(path, ".tfvars") +} + // IsIniFile returns true if a given file path corresponds to a INI file func IsIniFile(path string) bool { return strings.HasSuffix(path, ".ini") @@ -62,6 +69,8 @@ func FormatForPath(path string) Format { format = Dotenv } else if IsIniFile(path) { format = Ini + } else if IsHclFile(path) { + format = Hcl } return format } diff --git a/cmd/sops/formats/formats_test.go b/cmd/sops/formats/formats_test.go index 7bc6477fae..d247bfb05d 100644 --- a/cmd/sops/formats/formats_test.go +++ b/cmd/sops/formats/formats_test.go @@ -9,6 +9,7 @@ import ( func TestFormatFromString(t *testing.T) { assert.Equal(t, Binary, FormatFromString("foobar")) assert.Equal(t, Dotenv, FormatFromString("dotenv")) + assert.Equal(t, Hcl, FormatFromString("hcl")) assert.Equal(t, Ini, FormatFromString("ini")) assert.Equal(t, Yaml, FormatFromString("yaml")) assert.Equal(t, Json, FormatFromString("json")) @@ -17,6 +18,9 @@ func TestFormatFromString(t *testing.T) { func TestFormatForPath(t *testing.T) { assert.Equal(t, Binary, FormatForPath("/path/to/foobar")) assert.Equal(t, Dotenv, FormatForPath("/path/to/foobar.env")) + assert.Equal(t, Hcl, FormatForPath("/path/to/foobar.hcl")) + assert.Equal(t, Hcl, FormatForPath("/path/to/foobar.tfvars")) + assert.Equal(t, Json, FormatForPath("/path/to/foobar.tfvars.json")) assert.Equal(t, Ini, FormatForPath("/path/to/foobar.ini")) assert.Equal(t, Json, FormatForPath("/path/to/foobar.json")) assert.Equal(t, Yaml, FormatForPath("/path/to/foobar.yml")) @@ -27,6 +31,8 @@ func TestFormatForPathOrString(t *testing.T) { assert.Equal(t, Binary, FormatForPathOrString("/path/to/foobar", "")) assert.Equal(t, Dotenv, FormatForPathOrString("/path/to/foobar", "dotenv")) assert.Equal(t, Dotenv, FormatForPathOrString("/path/to/foobar.env", "")) + assert.Equal(t, Hcl, FormatForPathOrString("/path/to/foobar", "hcl")) + assert.Equal(t, Hcl, FormatForPathOrString("/path/to/foobar.tfvars", "")) assert.Equal(t, Ini, FormatForPathOrString("/path/to/foobar", "ini")) assert.Equal(t, Ini, FormatForPathOrString("/path/to/foobar.ini", "")) assert.Equal(t, Json, FormatForPathOrString("/path/to/foobar", "json")) diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 8092a3f400..0efd069c22 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -307,11 +307,11 @@ func main() { }, cli.StringFlag{ Name: "input-type", - Usage: "currently ini, json, yaml, dotenv and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently ini, json, yaml, dotenv, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently ini, json, yaml, dotenv and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently ini, json, yaml, dotenv, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.StringFlag{ Name: "filename", @@ -517,7 +517,7 @@ func main() { Flags: []cli.Flag{ cli.StringFlag{ Name: "input-type", - Usage: "currently ini, json, yaml, dotenv and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently ini, json, yaml, dotenv, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, }, Action: func(c *cli.Context) error { @@ -732,7 +732,7 @@ func main() { }, cli.StringFlag{ Name: "input-type", - Usage: "currently ini, json, yaml, dotenv and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently ini, json, yaml, dotenv, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, }, keyserviceFlags...), Action: func(c *cli.Context) error { @@ -801,11 +801,11 @@ func main() { }, cli.StringFlag{ Name: "input-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.BoolFlag{ Name: "ignore-mac", @@ -972,11 +972,11 @@ func main() { }, cli.StringFlag{ Name: "input-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.StringFlag{ Name: "unencrypted-suffix", @@ -1125,11 +1125,11 @@ func main() { }, cli.StringFlag{ Name: "input-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.StringFlag{ Name: "encryption-context", @@ -1331,11 +1331,11 @@ func main() { }, cli.StringFlag{ Name: "input-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.StringFlag{ Name: "unencrypted-suffix", @@ -1457,11 +1457,11 @@ func main() { Flags: append([]cli.Flag{ cli.StringFlag{ Name: "input-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.BoolFlag{ Name: "value-file", @@ -1593,11 +1593,11 @@ func main() { Flags: append([]cli.Flag{ cli.StringFlag{ Name: "input-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.IntFlag{ Name: "shamir-secret-sharing-threshold", @@ -1752,11 +1752,11 @@ func main() { }, cli.StringFlag{ Name: "input-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the file's extension to determine the type", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the file's extension to determine the type", }, cli.StringFlag{ Name: "output-type", - Usage: "currently json, yaml, dotenv, ini and binary are supported. If not set, sops will use the input file's extension to determine the output format", + Usage: "currently json, yaml, dotenv, ini, hcl and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, cli.BoolFlag{ Name: "show-master-keys, s", diff --git a/config/config.go b/config/config.go index 511df1bc15..65a98c8829 100644 --- a/config/config.go +++ b/config/config.go @@ -101,6 +101,8 @@ func FindConfigFile(start string) (string, error) { type DotenvStoreConfig struct{} +type HCLStoreConfig struct{} + type INIStoreConfig struct{} type JSONStoreConfig struct { @@ -117,6 +119,7 @@ type YAMLStoreConfig struct { type StoresConfig struct { Dotenv DotenvStoreConfig `yaml:"dotenv"` + HCL HCLStoreConfig `yaml:"hcl"` INI INIStoreConfig `yaml:"ini"` JSONBinary JSONBinaryStoreConfig `yaml:"json_binary"` JSON JSONStoreConfig `yaml:"json"` diff --git a/decrypt/decrypt.go b/decrypt/decrypt.go index e26cbe4798..98f7623f2b 100644 --- a/decrypt/decrypt.go +++ b/decrypt/decrypt.go @@ -72,7 +72,7 @@ func DataWithFormat(data []byte, format Format) (cleartext []byte, err error) { // Data is a helper that takes encrypted data and a format string, // decrypts the data and returns its cleartext in an []byte. -// The format string can be `json`, `yaml`, `ini`, `dotenv` or `binary`. +// The format string can be `json`, `yaml`, `ini`, `hcl`, `dotenv` or `binary`. // If the format string is empty, binary format is assumed. func Data(data []byte, format string) (cleartext []byte, err error) { formatFmt := FormatFromString(format) diff --git a/functional-tests/res/format.enc.hcl b/functional-tests/res/format.enc.hcl new file mode 100644 index 0000000000..1ca7e91c00 --- /dev/null +++ b/functional-tests/res/format.enc.hcl @@ -0,0 +1,23 @@ +foo = "ENC[AES256_GCM,data:VWTe,iv:1jXojBlsfxaGel2OlCAY9i0CDaaVRCudGjuJlV90EA8=,tag:yzj4lEr4OaccODNka2shBg==,type:str]" + +baz "qux" { + quux = [ + "ENC[AES256_GCM,data:fw==,iv:Qykwql3Kahp2uoSgTLUv60HGniPU839v2i2zoDPuxJ4=,tag:ZYU3fHRnO+spNg0+qJAH/A==,type:int]", + "ENC[AES256_GCM,data:MA==,iv:CDH2zKXaYMhJr4c/528Z/hFM2IfLxlvQSc1cgVH+WT4=,tag:HDpCS1qqOJDU64PCpUcVZQ==,type:int]", + ] + ref = "ENC[AES256_GCM,data:AVv1laLwAWU=,iv:E37lz0Sw+aFKC6sS5xtMELeu4a3DliYQR5z4Qs2Dox0=,tag:x/PC6Lfj7OpNdsployZLhg==,type:str]" +} + +sops = { + lastmodified = "2026-10-16T20:25:39Z" + mac = "ENC[AES256_GCM,data:tg9KJb7A0nHIb+SmlZiry1DDQop8JIiJtN405BLNs1oBD1y3zb1XJjve5k837K6Hm7xTVrb4gw/MgW5c32IyZdjTqEQv+FusdYZ3ElK3OcIFFUoNvbakOBnGBTNA+h28ViH5c6LaH8awVftfdguOxouODqMhvlD5a0mHLy38oAc=,iv:lct8P+0EXKLNg6dh6elZk4kM09dzaljC1fbnCc3vv24=,tag:E8Wu1fqMUSPiuQG4Yg1a4Q==,type:str]" + pgp = [ + { + created_at = "2026-10-16T20:25:39Z" + enc = "-----BEGIN PGP MESSAGE-----\n\nhQEMAyUpShfNkFB/AQf+MBQLTQghvVfi1n2Bu8F+0OP8WW7vx0ZKSUY19piKrh+G\nq8kfvpGkA+xHiQ3DGR034jeTwAcOBPnncJLF+omeeIUEO7frT9I4e7EMMBAKfubF\n5V/me5Oq85OhjvokzlxguTJeZvzgLll7KL4/vletIpUc2SHK9beK5dNiuVVSLjNc\nY4w9zM1t7SGQL/ueRmxCQBywAJsQbA8sclQHYI/0+C/fmjm1qEcYKdLwYDVCXSKB\nDOFhL4acuKu+eMljpxMe4vJ+dp+CQqF59WOKjYicDDSplYc2LVw9i7addD6NRRJJ\nQY7mfBFbATBfDyAAQdRINrbWpS05o/DdE41IQ/hl/tJeAbTDlkI0MI4U8p+LrERa\n6+VyeDX7tC/MNKzSznutBAbYnVRz4sZ4G5EBPJBVpSCcPEa8hmB+zcMaehjGwnFL\ndPXDtRMR4nUgzHi9Iha4OJFo5D+N5F2pFWFyqw1hyg==\n=0/eM\n-----END PGP MESSAGE-----" + fp = "FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4" + }, + ] + unencrypted_suffix = "_unencrypted" + version = "3.13.3" +} diff --git a/functional-tests/src/lib.rs b/functional-tests/src/lib.rs index 698d310516..3f0d243755 100644 --- a/functional-tests/src/lib.rs +++ b/functional-tests/src/lib.rs @@ -1737,6 +1737,24 @@ bar: |- String::from_utf8_lossy(&output.stdout) == "[foo]\nbar = baz\n", "Unexpected decrypted content" ); + + // HCL + let output = Command::new(SOPS_BINARY_PATH) + .arg("decrypt") + .arg("res/format.enc.hcl") + .output() + .expect("Error running sops"); + println!( + "stdout: {}, stderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.status.success(), "SOPS didn't return successfully"); + assert!( + String::from_utf8_lossy(&output.stdout) + == "foo = \"bar\"\n\nbaz \"qux\" {\n quux = [\n 1,\n 2,\n ]\n ref = var.x\n}\n", + "Unexpected decrypted content" + ); } #[test] diff --git a/go.mod b/go.mod index 811f12b2cd..35b416d1a5 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/goware/prefixer v0.0.0-20160118172347-395022866408 github.com/hashicorp/go-cleanhttp v0.5.2 + github.com/hashicorp/hcl/v2 v2.25.0 github.com/hashicorp/vault/api v1.23.0 github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.213 github.com/lib/pq v1.12.3 @@ -35,12 +36,14 @@ require ( github.com/sirupsen/logrus v1.10.2 github.com/stretchr/testify v1.12.1 github.com/urfave/cli v1.22.17 + github.com/zclconf/go-cty v1.19.0 go.yaml.in/yaml/v3 v3.0.5 golang.org/x/crypto v0.55.0 golang.org/x/net v0.58.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sys v0.47.0 golang.org/x/term v0.45.0 + golang.org/x/text v0.41.0 google.golang.org/api v0.294.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 google.golang.org/grpc v1.83.2 @@ -69,6 +72,9 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect + github.com/agext/levenshtein v1.2.1 // indirect + github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/apparentlymart/go-textseg/v17 v17.0.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.1 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1 // indirect @@ -148,9 +154,10 @@ require ( go.opentelemetry.io/otel/sdk v1.45.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect go.opentelemetry.io/otel/trace v1.45.0 // indirect + golang.org/x/mod v0.38.0 // indirect golang.org/x/sync v0.22.0 // indirect - golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect + golang.org/x/tools v0.48.0 // indirect google.golang.org/genproto v0.0.0-20260720171339-e059f2f05d78 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260720171339-e059f2f05d78 // indirect ) diff --git a/go.sum b/go.sum index 24b7aeb953..ade9ffe794 100644 --- a/go.sum +++ b/go.sum @@ -67,6 +67,12 @@ github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEV github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= +github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= +github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= +github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/apparentlymart/go-textseg/v17 v17.0.1 h1:bpMXRgQ5cEoRNuQke1a80/Nl6w3G5eoIbWo9f3gXkAs= +github.com/apparentlymart/go-textseg/v17 v17.0.1/go.mod h1:fa8X4jgGeevslICIY6LcdjkSecWnXmYd9Lk34z/VxZs= github.com/aws/aws-sdk-go-v2 v1.45.1 h1:iIoG3NaLhV6UZpPXyPXlDj2I9oS8tV/nMcMnITCC6Ks= github.com/aws/aws-sdk-go-v2 v1.45.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= @@ -239,6 +245,8 @@ github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9 github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= +github.com/hashicorp/hcl/v2 v2.25.0 h1:HmmQVYRny4MaBo4b20TjmL46wyuUxpnMWkPZ4+NTbWk= +github.com/hashicorp/hcl/v2 v2.25.0/go.mod h1:vR+FKETxoZAmRlHgFfKmuqivj+C4Izm/c66XkmZ3r7M= github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A= github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko= github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.213 h1:uKt4C7kPCCDk47+VLTPLUY/u4si2wr+SLFI7EEM8KSc= @@ -336,6 +344,10 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zclconf/go-cty v1.19.0 h1:IV8WdqYZc2c5rLX9bEoLNXKojBAp0MZPBHMIrCoa/s4= +github.com/zclconf/go-cty v1.19.0/go.mod h1:12W89jGn3JCOIQi7infWr9m80rOkb5RNYJqXMZcN4c8= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo= go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU= go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ= @@ -375,6 +387,8 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -434,6 +448,8 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/stores/hcl/store.go b/stores/hcl/store.go new file mode 100644 index 0000000000..7e63ba1c71 --- /dev/null +++ b/stores/hcl/store.go @@ -0,0 +1,536 @@ +/* +Package hcl implements a SOPS store for HCL (HashiCorp Configuration Language) +files such as Terraform `.tfvars` and Terragrunt `.hcl` files. + +Mapping between HCL and the SOPS tree: + + - An attribute becomes a tree item whose key is the attribute name. + - A block becomes a tree item whose key is the block header as it is written + in the file, including labels and the opening brace, for example + `resource "aws_instance" "web" {`. The trailing brace is what tells blocks + apart from attributes holding an object. The value is the block body. + - Literal strings, numbers and booleans become strings, ints/floats and bools, + and are encrypted. `null` becomes nil. + - Tuples and objects made only of literals become lists and mappings. + - Every other expression (references, function calls, conditionals, + templates with interpolation, ...) is opaque: its source text is wrapped as + `${}` and treated as a string. It is therefore encrypted like any + other string, and written back verbatim on decryption. This is the same + convention HCL's JSON syntax uses for expressions. + - Comments become sops.Comment items. +*/ +package hcl //import "github.com/getsops/sops/v3/stores/hcl" + +import ( + "bytes" + "encoding/json" + "fmt" + "math/big" + "sort" + "strconv" + "strings" + + "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" + "github.com/getsops/sops/v3/stores" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/zclconf/go-cty/cty" + "golang.org/x/text/unicode/norm" +) + +const indent = " " + +// Store handles storage of HCL data. +type Store struct { + config *config.HCLStoreConfig +} + +// NewStore creates a new HCL store. +func NewStore(c *config.HCLStoreConfig) *Store { + return &Store{config: c} +} + +// Name returns the name of the store. +func (store *Store) Name() string { + return "hcl" +} + +// isExpr reports whether a tree string stands for an opaque expression. +func isExpr(s string) bool { + return len(s) >= 3 && strings.HasPrefix(s, "${") && strings.HasSuffix(s, "}") +} + +func quote(s string) string { + return string(hclwrite.TokensForValue(cty.StringVal(s)).Bytes()) +} + +// --------------------------------------------------------------------------- +// HCL -> tree + +type span struct { + start, end int +} + +type comment struct { + span + text string +} + +type loader struct { + src []byte + comments []comment +} + +func (l *loader) hasComment(s span) bool { + for _, c := range l.comments { + if c.start >= s.start && c.end <= s.end { + return true + } + } + return false +} + +func commentText(raw string) string { + switch { + case strings.HasPrefix(raw, "/*"): + raw = strings.TrimSuffix(strings.TrimPrefix(raw, "/*"), "*/") + return strings.TrimSpace(raw) + case strings.HasPrefix(raw, "//"): + raw = raw[2:] + default: + raw = strings.TrimPrefix(raw, "#") + } + return strings.TrimPrefix(strings.TrimRight(raw, "\r\n"), " ") +} + +func (l *loader) opaque(e hclsyntax.Expression) string { + r := e.Range() + return "${" + string(l.src[r.Start.Byte:r.End.Byte]) + "}" +} + +func (l *loader) literal(s string) (interface{}, error) { + if isExpr(s) { + return nil, fmt.Errorf("string %q starts with ${ and ends with }, which cannot be told apart from an expression", s) + } + return s, nil +} + +func number(v cty.Value) (interface{}, bool) { + bf := v.AsBigFloat() + if bf.IsInt() { + if i, acc := bf.Int64(); acc == big.Exact && int64(int(i)) == i { + return int(i), true + } + } + f, _ := bf.Float64() + if r, err := cty.ParseNumberVal(strconv.FormatFloat(f, 'f', -1, 64)); err == nil && r.RawEquals(v) { + return f, true + } + return nil, false +} + +// nfc reports whether the source of e is NFC-normal. HCL normalizes decoded +// strings to NFC, which would change the bytes of anything that is not. +func (l *loader) nfc(r hcl.Range) bool { + src := l.src[r.Start.Byte:r.End.Byte] + // \u and \U escapes are ASCII in the source but can decode to non-NFC text. + return norm.NFC.IsNormal(src) && !bytes.Contains(src, []byte(`\u`)) && !bytes.Contains(src, []byte(`\U`)) +} + +func (l *loader) value(e hclsyntax.Expression) (interface{}, error) { + r := e.Range() + rs := span{r.Start.Byte, r.End.Byte} + switch e := e.(type) { + case *hclsyntax.UnaryOpExpr: + if lv, ok := e.Val.(*hclsyntax.LiteralValueExpr); ok && e.Op == hclsyntax.OpNegate && lv.Val.Type() == cty.Number { + if n, ok := number(lv.Val.Negate()); ok { + return n, nil + } + } + case *hclsyntax.LiteralValueExpr: + switch { + case e.Val.IsNull(): + return nil, nil + case e.Val.Type() == cty.Bool: + return e.Val.True(), nil + case e.Val.Type() == cty.Number: + if n, ok := number(e.Val); ok { + return n, nil + } + } + case *hclsyntax.TemplateExpr: + var sb strings.Builder + literal := true + for _, p := range e.Parts { + lv, ok := p.(*hclsyntax.LiteralValueExpr) + if !ok || lv.Val.Type() != cty.String { + literal = false + break + } + sb.WriteString(lv.Val.AsString()) + } + if literal && l.nfc(r) { + return l.literal(sb.String()) + } + case *hclsyntax.TupleConsExpr: + if l.hasComment(rs) { + break + } + out := make([]interface{}, 0, len(e.Exprs)) + for _, item := range e.Exprs { + v, err := l.value(item) + if err != nil { + return nil, err + } + out = append(out, v) + } + return out, nil + case *hclsyntax.ObjectConsExpr: + if l.hasComment(rs) { + break + } + out := sops.TreeBranch{} + ok := true + for _, item := range e.Items { + key := l.objectKey(item.KeyExpr) + if key == nil { + ok = false + break + } + v, err := l.value(item.ValueExpr) + if err != nil { + return nil, err + } + out = append(out, sops.TreeItem{Key: *key, Value: v}) + } + if ok { + return out, nil + } + } + return l.opaque(e), nil +} + +// objectKey returns the key of an object entry if it is a plain identifier or +// a literal string, and nil otherwise. +func (l *loader) objectKey(e hclsyntax.Expression) *string { + if kw := hcl.ExprAsKeyword(e); kw != "" { + return &kw + } + if w, ok := e.(*hclsyntax.ObjectConsKeyExpr); ok { + e = w.Wrapped + } + if t, ok := e.(*hclsyntax.TemplateExpr); ok && len(t.Parts) == 1 { + if lv, ok := t.Parts[0].(*hclsyntax.LiteralValueExpr); ok && lv.Val.Type() == cty.String { + if !l.nfc(t.Range()) { + return nil + } + s := lv.Val.AsString() + return &s + } + } + return nil +} + +type bodyItem struct { + span + attr *hclsyntax.Attribute + block *hclsyntax.Block +} + +// body converts the contents of body, which spans the bytes [lo, hi) of the +// source, to a tree branch. +func (l *loader) body(b *hclsyntax.Body, lo, hi int) (sops.TreeBranch, error) { + var items []bodyItem + for _, a := range b.Attributes { + items = append(items, bodyItem{span{a.SrcRange.Start.Byte, a.SrcRange.End.Byte}, a, nil}) + } + for _, blk := range b.Blocks { + items = append(items, bodyItem{span{blk.TypeRange.Start.Byte, blk.CloseBraceRange.End.Byte}, nil, blk}) + } + sort.Slice(items, func(i, j int) bool { return items[i].start < items[j].start }) + + var branch sops.TreeBranch + // addComments emits the comments located in [from, to). + addComments := func(from, to int, prev *bodyItem) { + for _, c := range l.comments { + if c.start < from || c.end > to { + continue + } + inline := prev != nil && prev.attr != nil && + !strings.Contains(string(l.src[prev.end:c.start]), "\n") + branch = append(branch, sops.TreeItem{Key: sops.Comment{Value: c.text, Inline: inline}}) + } + } + pos := lo + var prev *bodyItem + for i := range items { + it := &items[i] + addComments(pos, it.start, prev) + var ti sops.TreeItem + if it.attr != nil { + v, err := l.value(it.attr.Expr) + if err != nil { + return nil, fmt.Errorf("attribute %q: %w", it.attr.Name, err) + } + ti = sops.TreeItem{Key: it.attr.Name, Value: v} + } else { + blk := it.block + header := blk.Type + for i, label := range blk.Labels { + if !l.nfc(blk.LabelRanges[i]) { + return nil, fmt.Errorf("block label %q is not NFC-normalized, which HCL cannot represent", label) + } + header += " " + quote(label) + } + inner, err := l.body(blk.Body, blk.OpenBraceRange.End.Byte, blk.CloseBraceRange.Start.Byte) + if err != nil { + return nil, err + } + ti = sops.TreeItem{Key: header + " {", Value: inner} + } + branch = append(branch, ti) + pos = it.end + prev = it + } + addComments(pos, hi, prev) + return branch, nil +} + +// LoadPlainFile loads a plaintext HCL file's bytes onto a sops.TreeBranches runtime object +func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { + file, diags := hclsyntax.ParseConfig(in, "", hcl.InitialPos) + if diags.HasErrors() { + return nil, fmt.Errorf("Could not unmarshal input data: %s", diags.Error()) + } + tokens, diags := hclsyntax.LexConfig(in, "", hcl.InitialPos) + if diags.HasErrors() { + return nil, fmt.Errorf("Could not unmarshal input data: %s", diags.Error()) + } + l := &loader{src: in} + for _, t := range tokens { + if t.Type == hclsyntax.TokenComment { + l.comments = append(l.comments, comment{span{t.Range.Start.Byte, t.Range.End.Byte}, commentText(string(t.Bytes))}) + } + } + branch, err := l.body(file.Body.(*hclsyntax.Body), 0, len(in)) + if err != nil { + return nil, fmt.Errorf("Could not unmarshal input data: %s", err) + } + return sops.TreeBranches{branch}, nil +} + +// LoadEncryptedFile loads encrypted HCL file's bytes onto a sops.Tree runtime object +func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) { + branches, err := store.LoadPlainFile(in) + if err != nil { + return sops.Tree{}, err + } + branches, metadata, err := stores.ExtractMetadata(branches, stores.MetadataOpts{}) + if err != nil { + return sops.Tree{}, err + } + return sops.Tree{ + Branches: branches, + Metadata: metadata, + }, nil +} + +// --------------------------------------------------------------------------- +// tree -> HCL + +func isIdent(s string) bool { + return hclsyntax.ValidIdentifier(s) +} + +func scalar(v interface{}) (string, error) { + switch v := v.(type) { + case nil: + return "null", nil + case string: + if isExpr(v) { + return v[2 : len(v)-1], nil + } + return quote(v), nil + case []byte: + return quote(string(v)), nil + case bool: + return strconv.FormatBool(v), nil + case int: + return strconv.Itoa(v), nil + case float64: + if v != v || v > 1.7976931348623157e308 || v < -1.7976931348623157e308 { + return "", fmt.Errorf("cannot represent %v in HCL", v) + } + return strconv.FormatFloat(v, 'f', -1, 64), nil + } + return "", fmt.Errorf("unsupported value type %T", v) +} + +func writeComments(sb *strings.Builder, c sops.Comment, pad string) { + for _, line := range strings.Split(c.Value, "\n") { + sb.WriteString(pad + strings.TrimRight("# "+line, " ") + "\n") + } +} + +func writeValue(sb *strings.Builder, v interface{}, pad string) error { + switch v := v.(type) { + case sops.TreeBranch: + if len(v) == 0 { + sb.WriteString("{}") + return nil + } + sb.WriteString("{\n") + if err := writeItems(sb, v, pad+indent, false); err != nil { + return err + } + sb.WriteString(pad + "}") + case []interface{}: + if len(v) == 0 { + sb.WriteString("[]") + return nil + } + sb.WriteString("[\n") + for _, e := range v { + if c, ok := e.(sops.Comment); ok { + writeComments(sb, c, pad+indent) + continue + } + sb.WriteString(pad + indent) + if err := writeValue(sb, e, pad+indent); err != nil { + return err + } + sb.WriteString(",\n") + } + sb.WriteString(pad + "]") + default: + s, err := scalar(v) + if err != nil { + return err + } + sb.WriteString(s) + } + return nil +} + +func isBlock(key string) bool { + return strings.HasSuffix(key, "{") +} + +// writeItems writes the items of a branch. Blocks are only allowed if blocks is true. +func writeItems(sb *strings.Builder, branch sops.TreeBranch, pad string, blocks bool) error { + var pending []sops.Comment + prevBlock, first := false, true + // open reports that the last attribute line is still waiting for its newline, + // so that an inline comment can be appended to it. + open := false + closeLine := func() { + if open { + sb.WriteString("\n") + open = false + } + } + flush := func() { + for _, c := range pending { + writeComments(sb, c, pad) + } + pending = nil + } + for _, item := range branch { + if c, ok := item.Key.(sops.Comment); ok { + if c.Inline && open { + sb.WriteString(" # " + strings.ReplaceAll(c.Value, "\n", " ")) + closeLine() + } else { + closeLine() + pending = append(pending, c) + } + continue + } + closeLine() + key, ok := item.Key.(string) + if !ok { + return fmt.Errorf("unexpected key type %T", item.Key) + } + block := blocks && isBlock(key) + if !first && (block || prevBlock) { + sb.WriteString("\n") + } + first, prevBlock = false, block + flush() + if block { + sb.WriteString(pad + key + "\n") + inner, ok := item.Value.(sops.TreeBranch) + if !ok { + return fmt.Errorf("block %q: value must be a mapping", key) + } + if err := writeItems(sb, inner, pad+indent, true); err != nil { + return err + } + sb.WriteString(pad + "}\n") + continue + } + if !isIdent(key) { + key = quote(key) + } + sb.WriteString(pad + key + " = ") + if err := writeValue(sb, item.Value, pad); err != nil { + return fmt.Errorf("%s: %w", key, err) + } + open = true + } + closeLine() + flush() + return nil +} + +// EmitPlainFile returns the plaintext HCL file bytes corresponding to a sops.TreeBranches object +func (store *Store) EmitPlainFile(in sops.TreeBranches) ([]byte, error) { + var sb strings.Builder + for _, branch := range in { + if err := writeItems(&sb, branch, "", true); err != nil { + return nil, fmt.Errorf("Error marshaling to HCL: %s", err) + } + } + out := hclwrite.Format([]byte(sb.String())) + // Guard against values that would produce a syntactically invalid file. + if _, diags := hclsyntax.ParseConfig(out, "", hcl.InitialPos); diags.HasErrors() { + return nil, fmt.Errorf("Error marshaling to HCL: %s", diags.Error()) + } + return out, nil +} + +// EmitEncryptedFile returns encrypted HCL file bytes corresponding to a sops.Tree +// runtime object +func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { + branches, err := stores.SerializeMetadata(in, stores.MetadataOpts{}) + if err != nil { + return nil, fmt.Errorf("Error marshaling metadata: %s", err) + } + return store.EmitPlainFile(branches) +} + +// EmitValue returns a single value encoded in a generic interface{} as bytes +func (store *Store) EmitValue(v interface{}) ([]byte, error) { + switch v := v.(type) { + case sops.TreeBranches: + return store.EmitPlainFile(v) + default: + return json.Marshal(v) + } +} + +// EmitExample returns the plaintext HCL file bytes corresponding to the FlatTree example +func (store *Store) EmitExample() []byte { + bytes, err := store.EmitPlainFile(stores.ExampleFlatTree.Branches) + if err != nil { + panic(err) + } + return bytes +} + +// HasSopsTopLevelKey checks whether a top-level "sops" key exists. +func (store *Store) HasSopsTopLevelKey(branch sops.TreeBranch) bool { + return stores.HasSopsTopLevelKey(branch) +} diff --git a/stores/hcl/store_test.go b/stores/hcl/store_test.go new file mode 100644 index 0000000000..bf92878b5d --- /dev/null +++ b/stores/hcl/store_test.go @@ -0,0 +1,320 @@ +package hcl + +import ( + "strings" + "testing" + "time" + + "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/aes" + "github.com/getsops/sops/v3/age" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Inputs are in the format emitted by the store, so they must round-trip unchanged. +var roundTripCases = map[string]string{ + "attributes": `region = "eu-west-1" +count = 3 +ratio = 0.5 +enabled = true +disabled = false +nothing = null +`, + "lists and maps": `ports = [ + 80, + 443, +] +tags = { + env = "prod" + "with space" = "yes" + nested = { + a = 1 + } +} +empty_list = [] +empty_map = {} +mixed = [ + { + a = 1 + }, + [ + true, + ], +] +`, + "blocks": `variable "name" { + default = "x" +} + +resource "aws_instance" "web" { + ami = "ami-123" + + ebs_block_device { + size = 8 + } + + ebs_block_device { + size = 16 + } +} + +empty { +} +`, + "opaque expressions": `a = var.password +b = "prefix-${var.name}" +c = file("${path.module}/key.pem") +d = var.enabled ? "x" : "y" +e = lookup(var.m, "k", null) +f = [ + var.a, + "literal", +] +g = { + k = local.v +} +h = merge( + var.a, + { + k = "v" + }, +) +`, + "comments": `# leading +a = 1 # inline +# between +b = 2 + +# before block +block "x" { + # first in block + c = 3 # inline in block + # last in block +} +# trailing +`, + "escapes": `s = "line1\nline2 \"quoted\" $${not_interp} %%{not_directive}" +`, +} + +func TestRoundTrip(t *testing.T) { + for name, in := range roundTripCases { + t.Run(name, func(t *testing.T) { + store := &Store{} + branches, err := store.LoadPlainFile([]byte(in)) + require.NoError(t, err) + out, err := store.EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, in, string(out)) + }) + } +} + +func TestTreeShape(t *testing.T) { + in := `# top +password = "hunter2" # why +port = 8080 +ratio = 0.1 +ref = var.x +obj = { a = "b" } + +provider "aws" { + region = "us-east-1" +} +block { +} +` + branches, err := (&Store{}).LoadPlainFile([]byte(in)) + require.NoError(t, err) + assert.Equal(t, sops.TreeBranches{{ + {Key: sops.Comment{Value: "top"}}, + {Key: "password", Value: "hunter2"}, + {Key: sops.Comment{Value: "why", Inline: true}}, + {Key: "port", Value: 8080}, + {Key: "ratio", Value: 0.1}, + {Key: "ref", Value: "${var.x}"}, + {Key: "obj", Value: sops.TreeBranch{{Key: "a", Value: "b"}}}, + {Key: `provider "aws" {`, Value: sops.TreeBranch{{Key: "region", Value: "us-east-1"}}}, + {Key: "block {", Value: sops.TreeBranch(nil)}, + }}, branches) +} + +func TestBlockAndObjectAreDistinct(t *testing.T) { + in := "a {\n x = 1\n}\n\nb = {\n x = 1\n}\n" + store := &Store{} + branches, err := store.LoadPlainFile([]byte(in)) + require.NoError(t, err) + out, err := store.EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, in, string(out)) +} + +func TestCommentInsideLiteralStaysOpaque(t *testing.T) { + in := "l = [\n \"a\", # keep me\n \"b\",\n]\n" + store := &Store{} + branches, err := store.LoadPlainFile([]byte(in)) + require.NoError(t, err) + assert.Equal(t, "${[\n \"a\", # keep me\n \"b\",\n]}", branches[0][0].Value) + out, err := store.EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, in, string(out)) +} + +func TestBlockCommentsAreNormalized(t *testing.T) { + branches, err := (&Store{}).LoadPlainFile([]byte("/* one\ntwo */\n// three\na = 1\n")) + require.NoError(t, err) + out, err := (&Store{}).EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, "# one\n# two\n# three\na = 1\n", string(out)) +} + +func TestLiteralLookingLikeExpressionIsRejected(t *testing.T) { + _, err := (&Store{}).LoadPlainFile([]byte(`a = "$${x}"`)) + assert.ErrorContains(t, err, "cannot be told apart") +} + +func TestInvalidInput(t *testing.T) { + _, err := (&Store{}).LoadPlainFile([]byte(`a = = 1`)) + assert.Error(t, err) +} + +func TestEmitRejectsInvalidExpression(t *testing.T) { + _, err := (&Store{}).EmitPlainFile(sops.TreeBranches{{{Key: "a", Value: "${1 +}"}}}) + assert.Error(t, err) +} + +func TestEmitFromOtherFormats(t *testing.T) { + out, err := (&Store{}).EmitPlainFile(sops.TreeBranches{{ + {Key: "m", Value: sops.TreeBranch{{Key: "not an ident", Value: 1}}}, + {Key: "list", Value: []interface{}{"a", sops.TreeBranch{{Key: "k", Value: 2.5}}}}, + }}) + require.NoError(t, err) + assert.Equal(t, "m = {\n \"not an ident\" = 1\n}\nlist = [\n \"a\",\n {\n k = 2.5\n },\n]\n", string(out)) +} + +func TestNoMetadata(t *testing.T) { + _, err := (&Store{}).LoadEncryptedFile([]byte(`hello = 2`)) + assert.Equal(t, sops.MetadataNotFound, err) +} + +func TestHasSopsTopLevelKey(t *testing.T) { + store := &Store{} + assert.False(t, store.HasSopsTopLevelKey(sops.TreeBranch{{Key: "a", Value: "b"}})) + assert.True(t, store.HasSopsTopLevelKey(sops.TreeBranch{{Key: "sops", Value: "b"}})) +} + +// TestEncryptDecrypt runs a file through the real cipher: every literal and every +// opaque expression must be encrypted, metadata must survive the file, and +// decrypting must give back the input. +func TestEncryptDecrypt(t *testing.T) { + in := roundTripCases["opaque expressions"] + + "# leading\nport = 8080\nratio = 0.1\npassword = \"hunter2\"\nplain_unencrypted = \"visible\"\n" + + "resource \"a\" \"b\" {\n key = \"secret\"\n}\n" + in = string(mustFormat(t, in)) + store := &Store{} + branches, err := store.LoadPlainFile([]byte(in)) + require.NoError(t, err) + + dataKey := []byte("0123456789abcdef0123456789abcdef") + tree := sops.Tree{ + Branches: branches, + Metadata: sops.Metadata{ + UnencryptedSuffix: "_unencrypted", + LastModified: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + Version: "3.0.0", + KeyGroups: []sops.KeyGroup{{&age.MasterKey{ + Recipient: "age1yt3tfqlfrwdwx0z0ynwl3jvqmc5eg2zn2c4w5zcdlp0r92x2cwuq9n0qs8", + EncryptedKey: "-----BEGIN AGE ENCRYPTED FILE-----\nabc\n-----END AGE ENCRYPTED FILE-----\n", + }}}, + }, + } + cipher := aes.NewCipher() + mac, err := tree.Encrypt(dataKey, cipher) + require.NoError(t, err) + tree.Metadata.MessageAuthenticationCode, err = cipher.Encrypt(mac, dataKey, tree.Metadata.LastModified.Format(time.RFC3339)) + require.NoError(t, err) + + enc, err := store.EmitEncryptedFile(tree) + require.NoError(t, err) + for _, secret := range []string{"hunter2", "secret", "var.password", "var.name", "0.1", "leading"} { + assert.NotContains(t, strings.ReplaceAll(string(enc), "ENC[", ""), secret) + } + assert.Contains(t, string(enc), `plain_unencrypted = "visible"`) + assert.Contains(t, string(enc), `resource "a" "b" {`) + + loaded, err := store.LoadEncryptedFile(enc) + require.NoError(t, err) + assert.Equal(t, tree.Metadata.LastModified, loaded.Metadata.LastModified) + assert.Equal(t, tree.Metadata.UnencryptedSuffix, loaded.Metadata.UnencryptedSuffix) + require.Len(t, loaded.Metadata.KeyGroups, 1) + key := loaded.Metadata.KeyGroups[0][0].(*age.MasterKey) + assert.Equal(t, "-----BEGIN AGE ENCRYPTED FILE-----\nabc\n-----END AGE ENCRYPTED FILE-----\n", key.EncryptedKey) + + computed, err := loaded.Decrypt(dataKey, cipher) + require.NoError(t, err) + assert.Equal(t, mac, computed) + out, err := store.EmitPlainFile(loaded.Branches) + require.NoError(t, err) + assert.Equal(t, in, string(out)) +} + +func mustFormat(t *testing.T, in string) []byte { + branches, err := (&Store{}).LoadPlainFile([]byte(in)) + require.NoError(t, err) + out, err := (&Store{}).EmitPlainFile(branches) + require.NoError(t, err) + return out +} + +func TestEmitExample(t *testing.T) { + assert.NotEmpty(t, (&Store{}).EmitExample()) +} + +func TestNegativeNumbersKeepTheirType(t *testing.T) { + branches, err := (&Store{}).LoadPlainFile([]byte("i = -5\nf = -0.5\nx = -var.a\n")) + require.NoError(t, err) + assert.Equal(t, sops.TreeBranch{ + {Key: "i", Value: -5}, + {Key: "f", Value: -0.5}, + {Key: "x", Value: "${-var.a}"}, + }, branches[0]) + out, err := (&Store{}).EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, "i = -5\nf = -0.5\nx = -var.a\n", string(out)) +} + +func TestObjectKeyLookingLikeBlock(t *testing.T) { + in := "a = {\n \"foo {\" = \"x\"\n}\n" + store := &Store{} + branches, err := store.LoadPlainFile([]byte(in)) + require.NoError(t, err) + out, err := store.EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, in, string(out)) +} + +// HCL normalizes decoded strings to NFC; anything that is not NFC must be kept as source. +func TestNonNFCStringsAreKeptByteExact(t *testing.T) { + in := "s = \"café\"\nl = [\n \"café\",\n]\nm = {\n \"café\" = 1\n}\nn = \"café\"\n" + store := &Store{} + branches, err := store.LoadPlainFile([]byte(in)) + require.NoError(t, err) + assert.Equal(t, "${\"café\"}", branches[0][0].Value) + assert.Equal(t, "café", branches[0][3].Value) + out, err := store.EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, []byte(in), out) + + // The same text written with ASCII escapes must not be decoded either. + esc := "esc = \"e\\u0301\"\nobj = {\n \"e\\U00000301\" = 1\n}\n" + branches, err = store.LoadPlainFile([]byte(esc)) + require.NoError(t, err) + assert.Equal(t, "${\"e\\u0301\"}", branches[0][0].Value) + out, err = store.EmitPlainFile(branches) + require.NoError(t, err) + assert.Equal(t, esc, string(out)) + + _, err = store.LoadPlainFile([]byte("b \"café\" {\n}\n")) + assert.ErrorContains(t, err, "NFC") +}