Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 87 additions & 0 deletions parquet/encryption_read_config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,25 @@ package parquet_test

import (
"bytes"
"context"
"crypto/aes"
"crypto/cipher"
"encoding/base64"
"encoding/binary"
"encoding/json"
"fmt"
"os"
"path"
"strconv"
"testing"

"github.com/apache/arrow-go/v18/arrow/array"
"github.com/apache/arrow-go/v18/arrow/memory"
"github.com/apache/arrow-go/v18/parquet"
"github.com/apache/arrow-go/v18/parquet/file"
"github.com/apache/arrow-go/v18/parquet/internal/encryption"
"github.com/apache/arrow-go/v18/parquet/metadata"
"github.com/apache/arrow-go/v18/parquet/pqarrow"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
Expand Down Expand Up @@ -607,3 +615,82 @@ func (d *TestDecryptionSuite) TestDecryption() {
})
}
}

// javaKeyRetriever unwraps keys the way parquet-mr's KeyToolkit does for a file written
// with external key material and double wrapping: the KMS master key unwraps a key
// encryption key (KEK), and the KEK unwraps the data key.
type javaKeyRetriever struct {
t *testing.T
material map[string]string // key reference -> key material JSON
}

func (r javaKeyRetriever) unwrap(wrapped string, key, aad []byte) []byte {
raw, err := base64.StdEncoding.DecodeString(wrapped)
require.NoError(r.t, err)
block, err := aes.NewCipher(key)
require.NoError(r.t, err)
gcm, err := cipher.NewGCM(block)
require.NoError(r.t, err)
plain, err := gcm.Open(nil, raw[:gcm.NonceSize()], raw[gcm.NonceSize():], aad)
require.NoError(r.t, err)
return plain
}

func (r javaKeyRetriever) GetKey(keyMetadata []byte) string {
var ref struct{ KeyReference string }
require.NoError(r.t, json.Unmarshal(keyMetadata, &ref))
var km struct{ MasterKeyID, WrappedDEK, KeyEncryptionKeyID, WrappedKEK string }
require.NoError(r.t, json.Unmarshal([]byte(r.material[ref.KeyReference]), &km))

masterKeys := map[string]string{"kf": FooterEncryptionKey, "kc1": ColumnEncryptionKey1, "kc2": ColumnEncryptionKey2}
kek := r.unwrap(km.WrappedKEK, []byte(masterKeys[km.MasterKeyID]), []byte(km.MasterKeyID))
kekID, err := base64.StdEncoding.DecodeString(km.KeyEncryptionKeyID)
require.NoError(r.t, err)
return string(r.unwrap(km.WrappedDEK, kek, kekID))
}

// parquet-mr zero-pads every encrypted thrift module, page headers included, out to its
// buffer size, so reading its pages needs the reader to skip the whole encrypted header.
func TestDecryptParquetMrExternalKeyMaterial(t *testing.T) {
dir := getDataDir()
raw, err := os.ReadFile(path.Join(dir, "_KEY_MATERIAL_FOR_external_key_material_java.parquet.encrypted.json"))
require.NoError(t, err)
var material map[string]string
require.NoError(t, json.Unmarshal(raw, &material))

props := parquet.NewReaderProperties(memory.DefaultAllocator)
props.FileDecryptProps = parquet.NewFileDecryptionProperties(parquet.WithKeyRetriever(javaKeyRetriever{t, material}))
rdr, err := file.OpenParquetFile(path.Join(dir, "external_key_material_java.parquet.encrypted"), false, file.WithReadProps(props))
require.NoError(t, err)
defer rdr.Close()

fr, err := pqarrow.NewFileReader(rdr, pqarrow.ArrowReadProperties{}, memory.DefaultAllocator)
require.NoError(t, err)
tbl, err := fr.ReadTable(context.Background())
require.NoError(t, err)
defer tbl.Release()
require.EqualValues(t, 100, tbl.NumRows())
require.EqualValues(t, 2, tbl.NumCols())

// Same values the C++ ReadParquetMRExternalKeyMaterialFile test checks: column 0 is
// the row index and column 1 is prefixes[row % 10] followed by the row index.
prefixes := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}
row := 0
for _, chunk := range tbl.Column(0).Data().Chunks() {
ints := chunk.(*array.Int32)
for i := 0; i < ints.Len(); i++ {
assert.EqualValues(t, row, ints.Value(i), "integers[%d]", row)
row++
}
}
assert.Equal(t, 100, row)
row = 0
for _, chunk := range tbl.Column(1).Data().Chunks() {
strs := chunk.(*array.String)
for i := 0; i < strs.Len(); i++ {
assert.Equal(t, prefixes[row%len(prefixes)]+strconv.Itoa(row), strs.Value(i), "strings[%d]", row)
row++
}
}
assert.Equal(t, 100, row)
}
14 changes: 11 additions & 3 deletions parquet/file/page_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -805,7 +805,7 @@ func (p *serializedPageReader) readPageHeader(rd parquet.BufferedReader, hdr *fo
return io.EOF
}

extra := 0
framedLen := 0
if p.cryptoCtx.MetaDecryptor != nil {
p.updateDecryption(p.cryptoCtx.MetaDecryptor, encryption.DictPageHeaderModule, p.dataPageHeaderAad)
ciphertext, frameErr := encryption.FramedCiphertext(view)
Expand All @@ -823,7 +823,7 @@ func (p *serializedPageReader) readPageHeader(rd parquet.BufferedReader, hdr *fo
if err != nil {
return fmt.Errorf("decrypting page header: %w", err)
}
extra = p.cryptoCtx.MetaDecryptor.CiphertextSizeDelta()
framedLen = len(ciphertext)
}

remaining, err := thrift.DeserializeThrift(hdr, view)
Expand All @@ -834,7 +834,15 @@ func (p *serializedPageReader) readPageHeader(rd parquet.BufferedReader, hdr *fo
}
continue
}
rd.Discard(len(view) - int(remaining) + extra)
if p.cryptoCtx.MetaDecryptor != nil {
// An encrypted header is a length-prefixed module, so skip the whole frame.
// The decrypted payload can hold bytes after the thrift struct (parquet-mr
// zero-pads it to its buffer size), which means the amount of thrift read
// does not tell us how many bytes the module takes up in the file.
rd.Discard(framedLen)
} else {
rd.Discard(len(view) - int(remaining))
}
break
}
return nil
Expand Down
50 changes: 50 additions & 0 deletions parquet/file/page_reader_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,17 @@ package file

import (
"bytes"
"context"
"encoding/binary"
"io"
"math"
"testing"

"github.com/apache/arrow-go/v18/arrow/memory"
"github.com/apache/arrow-go/v18/parquet"
"github.com/apache/arrow-go/v18/parquet/internal/encryption"
format "github.com/apache/arrow-go/v18/parquet/internal/gen-go/parquet"
"github.com/apache/arrow-go/v18/parquet/internal/thrift"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -83,3 +88,48 @@ func TestReadLevelDataBitPacked(t *testing.T) {
require.Equal(t, data, out.Bytes())
out.Release()
}

// parquet-mr zero-pads every encrypted thrift module out to its buffer size, so a
// decrypted page header is the thrift struct followed by zero bytes. The reader has
// to move past the whole encrypted module, not just the thrift it parsed.
func TestReadEncryptedPageHeaderWithTrailingPadding(t *testing.T) {
const key = "0123456789012345"
encProps := parquet.NewFileEncryptionProperties(key)
enc := encryption.NewFileEncryptor(encProps, memory.DefaultAllocator).GetFooterEncryptor()
enc.UpdateAad(encryption.CreateModuleAad(encProps.FileAad(), encryption.DictPageHeaderModule, 0, 0, -1))

hdr := format.NewPageHeader()
hdr.Type = format.PageType_DICTIONARY_PAGE
hdr.CompressedPageSize, hdr.UncompressedPageSize = 10, 10
hdr.DictionaryPageHeader = format.NewDictionaryPageHeader()
thriftBytes, err := thrift.NewThriftSerializer().Write(context.Background(), hdr)
require.NoError(t, err)

// parquet-mr serializes each encrypted module into a TMemoryBuffer that starts at
// INIT_MEM_ALLOC_ENCR_BUFFER (100) bytes and encrypts the whole backing array, so a
// small header is followed by zeros up to 100 bytes. Pad the same way.
const parquetMrInitBuffer = 100
padding := make([]byte, parquetMrInitBuffer-len(thriftBytes))

var stream bytes.Buffer
enc.Encrypt(&stream, append(thriftBytes, padding...))
stream.WriteString("page data!")

dec := encryption.NewFileDecryptor(parquet.NewFileDecryptionProperties(parquet.WithFooterKey(key)),
encProps.FileAad(), parquet.AesGcm, "", memory.DefaultAllocator).GetFooterDecryptor()
rd, err := parquet.NewReaderProperties(memory.DefaultAllocator).
GetStream(bytes.NewReader(stream.Bytes()), 0, int64(stream.Len()))
require.NoError(t, err)

p := &serializedPageReader{
maxPageHeaderSize: defaultMaxPageHeaderSize,
cryptoCtx: CryptoContext{StartDecryptWithDictionaryPage: true, MetaDecryptor: dec},
}
got := format.NewPageHeader()
require.NoError(t, p.readPageHeader(rd, got))
require.Equal(t, format.PageType_DICTIONARY_PAGE, got.Type)

rest, err := io.ReadAll(rd)
require.NoError(t, err)
require.Equal(t, "page data!", string(rest))
}
Loading