@Override void setUp() throws Exception { HFileContextBuilder builder = new HFileContextBuilder() .withCompression(AbstractHFileWriter.compressionByName(codec)) .withBlockSize(RFILE_BLOCKSIZE); if (cipher == "aes") { byte[] cipherKey = new byte[AES.KEY_LENGTH]; new SecureRandom().nextBytes(cipherKey); builder.withEncryptionContext(Encryption.newContext(conf) .setCipher(Encryption.getCipher(conf, cipher)) .setKey(cipherKey)); } else if (!"none".equals(cipher)) { throw new IOException("Cipher " + cipher + " not supported."); } HFileContext hFileContext = builder.build(); writer = HFile.getWriterFactoryNoCache(conf) .withPath(fs, mf) .withFileContext(hFileContext) .withComparator(new KeyValue.RawBytesComparator()) .create(); }
@Test public void testWALKeyWrapping() throws Exception { // set up the key provider for testing to resolve a key for our test subject Configuration conf = new Configuration(); // we don't need HBaseConfiguration for this conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); // generate a test key byte[] keyBytes = new byte[AES.KEY_LENGTH]; new SecureRandom().nextBytes(keyBytes); String algorithm = conf.get(HConstants.CRYPTO_WAL_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); Key key = new SecretKeySpec(keyBytes, algorithm); // wrap the test key byte[] wrappedKeyBytes = EncryptionUtil.wrapKey(conf, "hbase", key); assertNotNull(wrappedKeyBytes); // unwrap Key unwrappedKey = EncryptionUtil.unwrapWALKey(conf, "hbase", wrappedKeyBytes); assertNotNull(unwrappedKey); // only secretkeyspec supported for now assertTrue(unwrappedKey instanceof SecretKeySpec); // did we get back what we wrapped? assertTrue("Unwrapped key bytes do not match original", Bytes.equals(keyBytes, unwrappedKey.getEncoded())); }
@Test(expected = KeyException.class) public void testWALKeyWrappingWithIncorrectKey() throws Exception { // set up the key provider for testing to resolve a key for our test subject Configuration conf = new Configuration(); // we don't need HBaseConfiguration for this conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); // generate a test key byte[] keyBytes = new byte[AES.KEY_LENGTH]; new SecureRandom().nextBytes(keyBytes); String algorithm = conf.get(HConstants.CRYPTO_WAL_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); Key key = new SecretKeySpec(keyBytes, algorithm); // wrap the test key byte[] wrappedKeyBytes = EncryptionUtil.wrapKey(conf, "hbase", key); assertNotNull(wrappedKeyBytes); // unwrap with an incorrect key EncryptionUtil.unwrapWALKey(conf, "other", wrappedKeyBytes); }
@Override void setUp() throws Exception { HFileContextBuilder builder = new HFileContextBuilder() .withCompression(HFileWriterImpl.compressionByName(codec)) .withBlockSize(RFILE_BLOCKSIZE); if (cipher == "aes") { byte[] cipherKey = new byte[AES.KEY_LENGTH]; new SecureRandom().nextBytes(cipherKey); builder.withEncryptionContext(Encryption.newContext(conf) .setCipher(Encryption.getCipher(conf, cipher)) .setKey(cipherKey)); } else if (!"none".equals(cipher)) { throw new IOException("Cipher " + cipher + " not supported."); } HFileContext hFileContext = builder.build(); writer = HFile.getWriterFactoryNoCache(conf) .withPath(fs, mf) .withFileContext(hFileContext) .withComparator(CellComparator.getInstance()) .create(); }
@Before public void setUp() throws Exception { conf = TEST_UTIL.getConfiguration(); conf.setInt("hfile.format.version", 3); conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); // Create the test encryption key SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[AES.KEY_LENGTH]; rng.nextBytes(keyBytes); String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); cfKey = new SecretKeySpec(keyBytes,algorithm); // Start the minicluster TEST_UTIL.startMiniCluster(3); // Create the table htd = new HTableDescriptor(TableName.valueOf("default", "TestHBaseFsckEncryption")); HColumnDescriptor hcd = new HColumnDescriptor("cf"); hcd.setEncryptionType(algorithm); hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey)); htd.addFamily(hcd); TEST_UTIL.getHBaseAdmin().createTable(htd); TEST_UTIL.waitTableAvailable(htd.getName(), 5000); }
@Test public void testKeyWrapping() throws Exception { // set up the key provider for testing to resolve a key for our test subject Configuration conf = new Configuration(); // we don't need HBaseConfiguration for this conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); // generate a test key byte[] keyBytes = new byte[AES.KEY_LENGTH]; new SecureRandom().nextBytes(keyBytes); String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); Key key = new SecretKeySpec(keyBytes, algorithm); // wrap the test key byte[] wrappedKeyBytes = EncryptionUtil.wrapKey(conf, "hbase", key); assertNotNull(wrappedKeyBytes); // unwrap Key unwrappedKey = EncryptionUtil.unwrapKey(conf, "hbase", wrappedKeyBytes); assertNotNull(unwrappedKey); // only secretkeyspec supported for now assertTrue(unwrappedKey instanceof SecretKeySpec); // did we get back what we wrapped? assertTrue("Unwrapped key bytes do not match original", Bytes.equals(keyBytes, unwrappedKey.getEncoded())); // unwrap with an incorrect key try { EncryptionUtil.unwrapKey(conf, "other", wrappedKeyBytes); fail("Unwrap with incorrect key did not throw KeyException"); } catch (KeyException e) { // expected } }
@Override public Cipher getCipher(String name) { if (name.equalsIgnoreCase("AES")) { return new AES(this); } throw new RuntimeException("Cipher '" + name + "' is not supported by provider '" + getName() + "'"); }
@Test public void testTestProvider() { Configuration conf = HBaseConfiguration.create(); conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); KeyProvider provider = Encryption.getKeyProvider(conf); assertNotNull("Null returned for provider", provider); assertTrue("Provider is not the expected type", provider instanceof KeyProviderForTesting); Key key = provider.getKey("foo"); assertNotNull("Test provider did not return a key as expected", key); assertEquals("Test provider did not create a key for AES", key.getAlgorithm(), "AES"); assertEquals("Test provider did not create a key of adequate length", key.getEncoded().length, AES.KEY_LENGTH); }
@Test public void testDefaultProvider() { Configuration conf = HBaseConfiguration.create(); CipherProvider provider = Encryption.getCipherProvider(conf); assertTrue(provider instanceof DefaultCipherProvider); String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); assertTrue(Arrays.asList(provider.getSupportedCiphers()).contains(algorithm)); Cipher a = Encryption.getCipher(conf, algorithm); assertNotNull(a); assertTrue(a.getProvider() instanceof DefaultCipherProvider); assertEquals(a.getName(), algorithm); assertEquals(a.getKeyLength(), AES.KEY_LENGTH); }
@Before public void setUp() throws Exception { conf = TEST_UTIL.getConfiguration(); conf.setInt("hfile.format.version", 3); conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); // Create the test encryption key SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[AES.KEY_LENGTH]; rng.nextBytes(keyBytes); cfKey = new SecretKeySpec(keyBytes, "AES"); // Start the minicluster TEST_UTIL.startMiniCluster(3); // Create the table htd = new HTableDescriptor(TableName.valueOf("default", "TestHBaseFsckEncryption")); HColumnDescriptor hcd = new HColumnDescriptor("cf"); hcd.setEncryptionType("AES"); hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey)); htd.addFamily(hcd); TEST_UTIL.getHBaseAdmin().createTable(htd); TEST_UTIL.waitTableAvailable(htd.getName(), 5000); }
@Test public void testKeyWrapping() throws Exception { // set up the key provider for testing to resolve a key for our test subject Configuration conf = new Configuration(); // we don't need HBaseConfiguration for this conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); // generate a test key byte[] keyBytes = new byte[AES.KEY_LENGTH]; new SecureRandom().nextBytes(keyBytes); Key key = new SecretKeySpec(keyBytes, "AES"); // wrap the test key byte[] wrappedKeyBytes = EncryptionUtil.wrapKey(conf, "hbase", key); assertNotNull(wrappedKeyBytes); // unwrap Key unwrappedKey = EncryptionUtil.unwrapKey(conf, "hbase", wrappedKeyBytes); assertNotNull(unwrappedKey); // only secretkeyspec supported for now assertTrue(unwrappedKey instanceof SecretKeySpec); // did we get back what we wrapped? assertTrue("Unwrapped key bytes do not match original", Bytes.equals(keyBytes, unwrappedKey.getEncoded())); // unwrap with an incorrect key try { EncryptionUtil.unwrapKey(conf, "other", wrappedKeyBytes); fail("Unwrap with incorrect key did not throw KeyException"); } catch (KeyException e) { // expected } }
@Test public void testDefaultProvider() { Configuration conf = HBaseConfiguration.create(); CipherProvider provider = Encryption.getCipherProvider(conf); assertTrue(provider instanceof DefaultCipherProvider); assertTrue(Arrays.asList(provider.getSupportedCiphers()).contains("AES")); Cipher a = Encryption.getCipher(conf, "AES"); assertNotNull(a); assertTrue(a.getProvider() instanceof DefaultCipherProvider); assertEquals(a.getName(), "AES"); assertEquals(a.getKeyLength(), AES.KEY_LENGTH); }
@Before public void setUp() throws Exception { conf = TEST_UTIL.getConfiguration(); conf.setInt("hfile.format.version", 3); conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); // Create the test encryption key SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[AES.KEY_LENGTH]; rng.nextBytes(keyBytes); String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); cfKey = new SecretKeySpec(keyBytes,algorithm); // Start the minicluster TEST_UTIL.startMiniCluster(3); // Create the table htd = new HTableDescriptor(TableName.valueOf("default", "TestHBaseFsckEncryption")); HColumnDescriptor hcd = new HColumnDescriptor("cf"); hcd.setEncryptionType(algorithm); hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey)); htd.addFamily(hcd); TEST_UTIL.getAdmin().createTable(htd); TEST_UTIL.waitTableAvailable(htd.getTableName(), 5000); }
@Test public void testTestProvider() { Configuration conf = HBaseConfiguration.create(); conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); KeyProvider provider = Encryption.getKeyProvider(conf); assertNotNull("Null returned for provider", provider); assertTrue("Provider is not the expected type", provider instanceof KeyProviderForTesting); Key key = provider.getKey("foo"); assertNotNull("Test provider did not return a key as expected", key); assertEquals("Test provider did not create a key for AES", "AES", key.getAlgorithm()); assertEquals("Test provider did not create a key of adequate length", AES.KEY_LENGTH, key.getEncoded().length); }
@Test public void testDefaultProvider() { Configuration conf = HBaseConfiguration.create(); CipherProvider provider = Encryption.getCipherProvider(conf); assertTrue(provider instanceof DefaultCipherProvider); String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); assertTrue(Arrays.asList(provider.getSupportedCiphers()).contains(algorithm)); Cipher a = Encryption.getCipher(conf, algorithm); assertNotNull(a); assertTrue(a.getProvider() instanceof DefaultCipherProvider); assertEquals(a.getName(), algorithm); assertEquals(AES.KEY_LENGTH, a.getKeyLength()); }
@Test(timeout = 300000) public void testMajorCompactionFromAdmin() throws Exception { resetConf(); int mergeSize = 5000; // change the mob compaction merge size conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize); SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[AES.KEY_LENGTH]; rng.nextBytes(keyBytes); String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); Key cfKey = new SecretKeySpec(keyBytes, algorithm); byte[] encryptionKey = EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey); final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor hcd1 = new HColumnDescriptor(family1); hcd1.setMobEnabled(true); hcd1.setMobThreshold(0); hcd1.setEncryptionType(algorithm); hcd1.setEncryptionKey(encryptionKey); HColumnDescriptor hcd2 = new HColumnDescriptor(family2); hcd2.setMobEnabled(true); hcd2.setMobThreshold(0); desc.addFamily(hcd1); desc.addFamily(hcd2); admin.createTable(desc, getSplitKeys()); Table table = conn.getTable(tableName); BufferedMutator bufMut = conn.getBufferedMutator(tableName); int count = 4; // generate mob files loadData(admin, bufMut, tableName, count, rowNumPerFile); int rowNumPerRegion = count * rowNumPerFile; assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion, countMobRows(table)); assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, countMobCells(table)); assertEquals("Before deleting: mob file count", regionNum * count, countFiles(tableName, true, family1)); createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), countMobRows(table)); assertEquals("Before compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("Before compaction: family1 mob file count", regionNum * count, countFiles(tableName, true, family1)); assertEquals("Before compaction: family2 mob file count", regionNum * count, countFiles(tableName, true, family2)); assertEquals("Before compaction: family1 del file count", regionNum, countFiles(tableName, false, family1)); assertEquals("Before compaction: family2 del file count", regionNum, countFiles(tableName, false, family2)); // do the major mob compaction, it will force all files to compaction admin.majorCompact(tableName, hcd1.getName(), CompactType.MOB); waitUntilMobCompactionFinished(tableName); assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), countMobRows(table)); assertEquals("After compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("After compaction: family1 mob file count", regionNum, countFiles(tableName, true, family1)); assertEquals("After compaction: family2 mob file count", regionNum * count, countFiles(tableName, true, family2)); assertEquals("After compaction: family1 del file count", 0, countFiles(tableName, false, family1)); assertEquals("After compaction: family2 del file count", regionNum, countFiles(tableName, false, family2)); Assert.assertTrue(verifyEncryption(tableName, family1)); table.close(); }
@Test public void testMOBStoreEncryption() throws Exception { final Configuration conf = TEST_UTIL.getConfiguration(); conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[AES.KEY_LENGTH]; rng.nextBytes(keyBytes); String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); Key cfKey = new SecretKeySpec(keyBytes, algorithm); HColumnDescriptor hcd = new HColumnDescriptor(family); hcd.setMobEnabled(true); hcd.setMobThreshold(100); hcd.setMaxVersions(4); hcd.setEncryptionType(algorithm); hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()),cfKey)); init(name.getMethodName(), conf, hcd, false); this.store.add(new KeyValue(row, family, qf1, 1, value), null); this.store.add(new KeyValue(row, family, qf2, 1, value), null); this.store.add(new KeyValue(row, family, qf3, 1, value), null); flush(1); this.store.add(new KeyValue(row, family, qf4, 1, value), null); this.store.add(new KeyValue(row, family, qf5, 1, value), null); this.store.add(new KeyValue(row, family, qf6, 1, value), null); flush(2); Collection<HStoreFile> storefiles = this.store.getStorefiles(); checkMobHFileEncrytption(storefiles); // Scan the values Scan scan = new Scan(get); InternalScanner scanner = (InternalScanner) store.getScanner(scan, scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0); List<Cell> results = new ArrayList<>(); scanner.next(results); Collections.sort(results, CellComparatorImpl.COMPARATOR); scanner.close(); Assert.assertEquals(expected.size(), results.size()); for(int i=0; i<results.size(); i++) { Assert.assertEquals(expected.get(i), results.get(i)); } // Trigger major compaction this.store.triggerMajorCompaction(); Optional<CompactionContext> requestCompaction = this.store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null); this.store.compact(requestCompaction.get(), NoLimitThroughputController.INSTANCE, null); Assert.assertEquals(1, this.store.getStorefiles().size()); //Check encryption after compaction checkMobHFileEncrytption(this.store.getStorefiles()); }