public PartitionedMobCompactor(Configuration conf, FileSystem fs, TableName tableName, ColumnFamilyDescriptor column, ExecutorService pool) throws IOException { super(conf, fs, tableName, column, pool); mergeableSize = conf.getLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD); delFileMaxCount = conf.getInt(MobConstants.MOB_DELFILE_MAX_COUNT, MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT); // default is 100 compactionBatchSize = conf.getInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE); tempPath = new Path(MobUtils.getMobHome(conf), MobConstants.TEMP_DIR_NAME); bulkloadPath = new Path(tempPath, new Path(MobConstants.BULKLOAD_DIR_NAME, new Path( tableName.getNamespaceAsString(), tableName.getQualifierAsString()))); compactionKVMax = this.conf.getInt(HConstants.COMPACTION_KV_MAX, HConstants.COMPACTION_KV_MAX_DEFAULT); Configuration copyOfConf = new Configuration(conf); copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f); compactionCacheConfig = new CacheConfig(copyOfConf); List<Tag> tags = new ArrayList<>(2); tags.add(MobConstants.MOB_REF_TAG); Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, tableName.getName()); tags.add(tableNameTag); this.refCellTags = TagUtil.fromList(tags); cryptoContext = EncryptionUtil.createEncryptionContext(copyOfConf, column); }
protected void finishClose(FixedFileTrailer trailer) throws IOException { // Write out encryption metadata before finalizing if we have a valid crypto context Encryption.Context cryptoContext = hFileContext.getEncryptionContext(); if (cryptoContext != Encryption.Context.NONE) { // Wrap the context's key and write it as the encryption metadata, the wrapper includes // all information needed for decryption trailer.setEncryptionKey(EncryptionUtil.wrapKey(cryptoContext.getConf(), cryptoContext.getConf().get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cryptoContext.getKey())); } // Now we can finish the close trailer.setMetaIndexCount(metaNames.size()); trailer.setTotalUncompressedBytes(totalUncompressedBytes+ trailer.getTrailerSize()); trailer.setEntryCount(entryCount); trailer.setCompressionCodec(hFileContext.getCompression()); long startTime = System.currentTimeMillis(); trailer.serialize(outputStream); HFile.updateWriteLatency(System.currentTimeMillis() - startTime); if (closeOutputStream) { outputStream.close(); outputStream = null; } }
@Override protected WALHeader buildWALHeader(Configuration conf, WALHeader.Builder builder) throws IOException { builder.setWriterClsName(SecureProtobufLogWriter.class.getSimpleName()); if (conf.getBoolean(HConstants.ENABLE_WAL_ENCRYPTION, false)) { EncryptionTest.testKeyProvider(conf); EncryptionTest.testCipherProvider(conf); // Get an instance of our cipher final String cipherName = conf.get(HConstants.CRYPTO_WAL_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); Cipher cipher = Encryption.getCipher(conf, cipherName); if (cipher == null) { throw new RuntimeException("Cipher '" + cipherName + "' is not available"); } // Generate an encryption key for this WAL SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[cipher.getKeyLength()]; rng.nextBytes(keyBytes); Key key = new SecretKeySpec(keyBytes, cipher.getName()); builder.setEncryptionKey(ByteStringer.wrap(EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_WAL_KEY_NAME_CONF_KEY, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName())), key))); // Set up the encryptor encryptor = cipher.getEncryptor(); encryptor.setKey(key); if (LOG.isTraceEnabled()) { LOG.trace("Initialized secure protobuf WAL: cipher=" + cipher.getName()); } } builder.setCellCodecClsName(SecureWALCellCodec.class.getName()); return super.buildWALHeader(conf, builder); }
@Override protected void finishClose(FixedFileTrailer trailer) throws IOException { // Write out encryption metadata before finalizing if we have a valid crypto context Encryption.Context cryptoContext = hFileContext.getEncryptionContext(); if (cryptoContext != Encryption.Context.NONE) { // Wrap the context's key and write it as the encryption metadata, the wrapper includes // all information needed for decryption trailer.setEncryptionKey(EncryptionUtil.wrapKey(cryptoContext.getConf(), cryptoContext.getConf().get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cryptoContext.getKey())); } // Now we can finish the close super.finishClose(trailer); }
@Before public void setUp() throws Exception { conf = TEST_UTIL.getConfiguration(); conf.setInt("hfile.format.version", 3); conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); // Create the test encryption key SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[AES.KEY_LENGTH]; rng.nextBytes(keyBytes); String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); cfKey = new SecretKeySpec(keyBytes,algorithm); // Start the minicluster TEST_UTIL.startMiniCluster(3); // Create the table htd = new HTableDescriptor(TableName.valueOf("default", "TestHBaseFsckEncryption")); HColumnDescriptor hcd = new HColumnDescriptor("cf"); hcd.setEncryptionType(algorithm); hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey)); htd.addFamily(hcd); TEST_UTIL.getHBaseAdmin().createTable(htd); TEST_UTIL.waitTableAvailable(htd.getName(), 5000); }
@Override protected WALHeader buildWALHeader(Configuration conf, WALHeader.Builder builder) throws IOException { builder.setWriterClsName(SecureProtobufLogWriter.class.getSimpleName()); if (conf.getBoolean(HConstants.ENABLE_WAL_ENCRYPTION, false)) { EncryptionTest.testKeyProvider(conf); EncryptionTest.testCipherProvider(conf); // Get an instance of our cipher final String cipherName = conf.get(HConstants.CRYPTO_WAL_ALGORITHM_CONF_KEY, DEFAULT_CIPHER); Cipher cipher = Encryption.getCipher(conf, cipherName); if (cipher == null) { throw new RuntimeException("Cipher '" + cipherName + "' is not available"); } // Generate an encryption key for this WAL SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[cipher.getKeyLength()]; rng.nextBytes(keyBytes); Key key = new SecretKeySpec(keyBytes, cipher.getName()); builder.setEncryptionKey(ByteStringer.wrap(EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_WAL_KEY_NAME_CONF_KEY, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName())), key))); // Set up the encryptor encryptor = cipher.getEncryptor(); encryptor.setKey(key); if (LOG.isTraceEnabled()) { LOG.trace("Initialized secure protobuf WAL: cipher=" + cipher.getName()); } } builder.setCellCodecClsName(SecureWALCellCodec.class.getName()); return super.buildWALHeader(conf, builder); }
@Before public void setUp() throws Exception { conf = TEST_UTIL.getConfiguration(); conf.setInt("hfile.format.version", 3); conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); // Create the test encryption key SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[AES.KEY_LENGTH]; rng.nextBytes(keyBytes); cfKey = new SecretKeySpec(keyBytes, "AES"); // Start the minicluster TEST_UTIL.startMiniCluster(3); // Create the table htd = new HTableDescriptor(TableName.valueOf("default", "TestHBaseFsckEncryption")); HColumnDescriptor hcd = new HColumnDescriptor("cf"); hcd.setEncryptionType("AES"); hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey)); htd.addFamily(hcd); TEST_UTIL.getHBaseAdmin().createTable(htd); TEST_UTIL.waitTableAvailable(htd.getName(), 5000); }
@Override protected WALHeader buildWALHeader(WALHeader.Builder builder) throws IOException { if (conf.getBoolean(HConstants.ENABLE_WAL_ENCRYPTION, false)) { // Get an instance of our cipher Cipher cipher = Encryption.getCipher(conf, conf.get(HConstants.CRYPTO_WAL_ALGORITHM_CONF_KEY, DEFAULT_CIPHER)); if (cipher == null) { throw new RuntimeException("Cipher '" + cipher + "' is not available"); } // Generate an encryption key for this WAL SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[cipher.getKeyLength()]; rng.nextBytes(keyBytes); Key key = new SecretKeySpec(keyBytes, cipher.getName()); builder.setEncryptionKey(HBaseZeroCopyByteString.wrap(EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_WAL_KEY_NAME_CONF_KEY, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName())), key))); // Set up the encryptor encryptor = cipher.getEncryptor(); encryptor.setKey(key); if (LOG.isTraceEnabled()) { LOG.trace("Initialized secure protobuf WAL: cipher=" + cipher.getName()); } } return super.buildWALHeader(builder); }
@Test public void testKeyWrapping() throws Exception { // set up the key provider for testing to resolve a key for our test subject Configuration conf = new Configuration(); // we don't need HBaseConfiguration for this conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); // generate a test key byte[] keyBytes = new byte[AES.KEY_LENGTH]; new SecureRandom().nextBytes(keyBytes); Key key = new SecretKeySpec(keyBytes, "AES"); // wrap the test key byte[] wrappedKeyBytes = EncryptionUtil.wrapKey(conf, "hbase", key); assertNotNull(wrappedKeyBytes); // unwrap Key unwrappedKey = EncryptionUtil.unwrapKey(conf, "hbase", wrappedKeyBytes); assertNotNull(unwrappedKey); // only secretkeyspec supported for now assertTrue(unwrappedKey instanceof SecretKeySpec); // did we get back what we wrapped? assertTrue("Unwrapped key bytes do not match original", Bytes.equals(keyBytes, unwrappedKey.getEncoded())); // unwrap with an incorrect key try { EncryptionUtil.unwrapKey(conf, "other", wrappedKeyBytes); fail("Unwrap with incorrect key did not throw KeyException"); } catch (KeyException e) { // expected } }
protected final WALHeader buildSecureWALHeader(Configuration conf, WALHeader.Builder builder) throws IOException { builder.setWriterClsName(getWriterClassName()); if (conf.getBoolean(HConstants.ENABLE_WAL_ENCRYPTION, false)) { EncryptionTest.testKeyProvider(conf); EncryptionTest.testCipherProvider(conf); // Get an instance of our cipher final String cipherName = conf.get(HConstants.CRYPTO_WAL_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); Cipher cipher = Encryption.getCipher(conf, cipherName); if (cipher == null) { throw new RuntimeException("Cipher '" + cipherName + "' is not available"); } // Generate an encryption key for this WAL SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[cipher.getKeyLength()]; rng.nextBytes(keyBytes); Key key = new SecretKeySpec(keyBytes, cipher.getName()); builder.setEncryptionKey(UnsafeByteOperations.unsafeWrap(EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_WAL_KEY_NAME_CONF_KEY, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName())), key))); // Set up the encryptor Encryptor encryptor = cipher.getEncryptor(); encryptor.setKey(key); setEncryptor(encryptor); if (LOG.isTraceEnabled()) { LOG.trace("Initialized secure protobuf WAL: cipher=" + cipher.getName()); } } builder.setCellCodecClsName(SecureWALCellCodec.class.getName()); return buildWALHeader0(conf, builder); }
protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize, HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException { HFileContextBuilder builder = new HFileContextBuilder() .withIncludesMvcc(shouldIncludeMemStoreTS()) .withHBaseCheckSum(true) .withHFileName(this.getName()) .withCompression(this.compressAlgo); // Check for any key material available byte[] keyBytes = trailer.getEncryptionKey(); if (keyBytes != null) { Encryption.Context cryptoContext = Encryption.newContext(conf); Key key; key = EncryptionUtil.unwrapKey(conf, keyBytes); // Use the algorithm the key wants Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm()); if (cipher == null) { throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available"); } cryptoContext.setCipher(cipher); cryptoContext.setKey(key); builder.withEncryptionContext(cryptoContext); } HFileContext context = builder.build(); if (LOG.isTraceEnabled()) { LOG.trace("Reader" + (path != null? " for " + path: "") + " initialized with cacheConf: " + cacheConf + " comparator: " + comparator.getClass().getSimpleName() + " fileContext: " + context); } return context; }
@Before public void setUp() throws Exception { conf = TEST_UTIL.getConfiguration(); conf.setInt("hfile.format.version", 3); conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); // Create the test encryption key SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[AES.KEY_LENGTH]; rng.nextBytes(keyBytes); String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); cfKey = new SecretKeySpec(keyBytes,algorithm); // Start the minicluster TEST_UTIL.startMiniCluster(3); // Create the table htd = new HTableDescriptor(TableName.valueOf("default", "TestHBaseFsckEncryption")); HColumnDescriptor hcd = new HColumnDescriptor("cf"); hcd.setEncryptionType(algorithm); hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey)); htd.addFamily(hcd); TEST_UTIL.getAdmin().createTable(htd); TEST_UTIL.waitTableAvailable(htd.getTableName(), 5000); }
/** * Check that the specified cipher can be loaded and initialized, or throw * an exception. Verifies key and cipher provider configuration as a * prerequisite for cipher verification. * * @param conf * @param cipher * @param key * @throws IOException */ public static void testEncryption(final Configuration conf, final String cipher, byte[] key) throws IOException { if (cipher == null) { return; } testKeyProvider(conf); testCipherProvider(conf); Boolean result = cipherResults.get(cipher); if (result == null) { try { Encryption.Context context = Encryption.newContext(conf); context.setCipher(Encryption.getCipher(conf, cipher)); if (key == null) { // Make a random key since one was not provided context.setKey(context.getCipher().getRandomKey()); } else { // This will be a wrapped key from schema context.setKey(EncryptionUtil.unwrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"), key)); } byte[] iv = null; if (context.getCipher().getIvLength() > 0) { iv = new byte[context.getCipher().getIvLength()]; Bytes.random(iv); } byte[] plaintext = new byte[1024]; Bytes.random(plaintext); ByteArrayOutputStream out = new ByteArrayOutputStream(); Encryption.encrypt(out, new ByteArrayInputStream(plaintext), context, iv); byte[] ciphertext = out.toByteArray(); out.reset(); Encryption.decrypt(out, new ByteArrayInputStream(ciphertext), plaintext.length, context, iv); byte[] test = out.toByteArray(); if (!Bytes.equals(plaintext, test)) { throw new IOException("Did not pass encrypt/decrypt test"); } cipherResults.put(cipher, true); } catch (Exception e) { cipherResults.put(cipher, false); throw new IOException("Cipher " + cipher + " failed test: " + e.getMessage(), e); } } else if (result.booleanValue() == false) { throw new IOException("Cipher " + cipher + " previously failed test"); } }
@Override protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize, HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException { trailer.expectMajorVersion(3); HFileContextBuilder builder = new HFileContextBuilder() .withIncludesMvcc(shouldIncludeMemstoreTS()) .withHBaseCheckSum(true) .withCompression(this.compressAlgo); // Check for any key material available byte[] keyBytes = trailer.getEncryptionKey(); if (keyBytes != null) { Encryption.Context cryptoContext = Encryption.newContext(conf); Key key; String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()); try { // First try the master key key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes); } catch (KeyException e) { // If the current master key fails to unwrap, try the alternate, if // one is configured if (LOG.isDebugEnabled()) { LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'"); } String alternateKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY); if (alternateKeyName != null) { try { key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes); } catch (KeyException ex) { throw new IOException(ex); } } else { throw new IOException(e); } } // Use the algorithm the key wants Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm()); if (cipher == null) { throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available"); } cryptoContext.setCipher(cipher); cryptoContext.setKey(key); builder.withEncryptionContext(cryptoContext); } HFileContext context = builder.build(); if (LOG.isTraceEnabled()) { LOG.trace("Reader" + (path != null ? " for " + path : "" ) + " initialized with cacheConf: " + cacheConf + " comparator: " + comparator.getClass().getSimpleName() + " fileContext: " + context); } return context; }
@Override protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize, HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException { trailer.expectMajorVersion(3); HFileContextBuilder builder = new HFileContextBuilder() .withIncludesMvcc(this.includesMemstoreTS) .withHBaseCheckSum(true) .withCompression(this.compressAlgo); // Check for any key material available byte[] keyBytes = trailer.getEncryptionKey(); if (keyBytes != null) { Encryption.Context cryptoContext = Encryption.newContext(conf); Key key; String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()); try { // First try the master key key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes); } catch (KeyException e) { // If the current master key fails to unwrap, try the alternate, if // one is configured if (LOG.isDebugEnabled()) { LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'"); } String alternateKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY); if (alternateKeyName != null) { try { key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes); } catch (KeyException ex) { throw new IOException(ex); } } else { throw new IOException(e); } } // Use the algorithm the key wants Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm()); if (cipher == null) { throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available"); } cryptoContext.setCipher(cipher); cryptoContext.setKey(key); builder.withEncryptionContext(cryptoContext); } HFileContext context = builder.build(); if (LOG.isTraceEnabled()) { LOG.trace("Reader" + (path != null ? " for " + path : "" ) + " initialized with cacheConf: " + cacheConf + " comparator: " + comparator.getClass().getSimpleName() + " fileContext: " + context); } return context; }
/** * Apply column family options such as Bloom filters, compression, and data * block encoding. */ protected void applyColumnFamilyOptions(TableName tableName, byte[][] columnFamilies) throws IOException { Admin admin = new HBaseAdmin(conf); HTableDescriptor tableDesc = admin.getTableDescriptor(tableName); LOG.info("Disabling table " + tableName); admin.disableTable(tableName); for (byte[] cf : columnFamilies) { HColumnDescriptor columnDesc = tableDesc.getFamily(cf); boolean isNewCf = columnDesc == null; if (isNewCf) { columnDesc = new HColumnDescriptor(cf); } if (bloomType != null) { columnDesc.setBloomFilterType(bloomType); } if (compressAlgo != null) { columnDesc.setCompressionType(compressAlgo); } if (dataBlockEncodingAlgo != null) { columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo); } if (inMemoryCF) { columnDesc.setInMemory(inMemoryCF); } if (cipher != null) { byte[] keyBytes = new byte[cipher.getKeyLength()]; new SecureRandom().nextBytes(keyBytes); columnDesc.setEncryptionType(cipher.getName()); columnDesc.setEncryptionKey(EncryptionUtil.wrapKey(conf, User.getCurrent().getShortName(), new SecretKeySpec(keyBytes, cipher.getName()))); } if (isNewCf) { admin.addColumn(tableName, columnDesc); } else { admin.modifyColumn(tableName, columnDesc); } } LOG.info("Enabling table " + tableName); admin.enableTable(tableName); admin.close(); }
/** * Apply column family options such as Bloom filters, compression, and data * block encoding. */ protected void applyColumnFamilyOptions(TableName tableName, byte[][] columnFamilies) throws IOException { HBaseAdmin admin = new HBaseAdmin(conf); HTableDescriptor tableDesc = admin.getTableDescriptor(tableName); LOG.info("Disabling table " + tableName); admin.disableTable(tableName); for (byte[] cf : columnFamilies) { HColumnDescriptor columnDesc = tableDesc.getFamily(cf); boolean isNewCf = columnDesc == null; if (isNewCf) { columnDesc = new HColumnDescriptor(cf); } if (bloomType != null) { columnDesc.setBloomFilterType(bloomType); } if (compressAlgo != null) { columnDesc.setCompressionType(compressAlgo); } if (dataBlockEncodingAlgo != null) { columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo); } if (inMemoryCF) { columnDesc.setInMemory(inMemoryCF); } if (cipher != null) { byte[] keyBytes = new byte[cipher.getKeyLength()]; new SecureRandom().nextBytes(keyBytes); columnDesc.setEncryptionType(cipher.getName()); columnDesc.setEncryptionKey(EncryptionUtil.wrapKey(conf, User.getCurrent().getShortName(), new SecretKeySpec(keyBytes, cipher.getName()))); } if (isNewCf) { admin.addColumn(tableName, columnDesc); } else { admin.modifyColumn(tableName, columnDesc); } } LOG.info("Enabling table " + tableName); admin.enableTable(tableName); }
@Test(timeout = 300000) public void testMajorCompactionFromAdmin() throws Exception { resetConf(); int mergeSize = 5000; // change the mob compaction merge size conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize); SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[AES.KEY_LENGTH]; rng.nextBytes(keyBytes); String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); Key cfKey = new SecretKeySpec(keyBytes, algorithm); byte[] encryptionKey = EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey); final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor hcd1 = new HColumnDescriptor(family1); hcd1.setMobEnabled(true); hcd1.setMobThreshold(0); hcd1.setEncryptionType(algorithm); hcd1.setEncryptionKey(encryptionKey); HColumnDescriptor hcd2 = new HColumnDescriptor(family2); hcd2.setMobEnabled(true); hcd2.setMobThreshold(0); desc.addFamily(hcd1); desc.addFamily(hcd2); admin.createTable(desc, getSplitKeys()); Table table = conn.getTable(tableName); BufferedMutator bufMut = conn.getBufferedMutator(tableName); int count = 4; // generate mob files loadData(admin, bufMut, tableName, count, rowNumPerFile); int rowNumPerRegion = count * rowNumPerFile; assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion, countMobRows(table)); assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, countMobCells(table)); assertEquals("Before deleting: mob file count", regionNum * count, countFiles(tableName, true, family1)); createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), countMobRows(table)); assertEquals("Before compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("Before compaction: family1 mob file count", regionNum * count, countFiles(tableName, true, family1)); assertEquals("Before compaction: family2 mob file count", regionNum * count, countFiles(tableName, true, family2)); assertEquals("Before compaction: family1 del file count", regionNum, countFiles(tableName, false, family1)); assertEquals("Before compaction: family2 del file count", regionNum, countFiles(tableName, false, family2)); // do the major mob compaction, it will force all files to compaction admin.majorCompact(tableName, hcd1.getName(), CompactType.MOB); waitUntilMobCompactionFinished(tableName); assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), countMobRows(table)); assertEquals("After compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("After compaction: family1 mob file count", regionNum, countFiles(tableName, true, family1)); assertEquals("After compaction: family2 mob file count", regionNum * count, countFiles(tableName, true, family2)); assertEquals("After compaction: family1 del file count", 0, countFiles(tableName, false, family1)); assertEquals("After compaction: family2 del file count", regionNum, countFiles(tableName, false, family2)); Assert.assertTrue(verifyEncryption(tableName, family1)); table.close(); }
@Test public void testMOBStoreEncryption() throws Exception { final Configuration conf = TEST_UTIL.getConfiguration(); conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[AES.KEY_LENGTH]; rng.nextBytes(keyBytes); String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); Key cfKey = new SecretKeySpec(keyBytes, algorithm); HColumnDescriptor hcd = new HColumnDescriptor(family); hcd.setMobEnabled(true); hcd.setMobThreshold(100); hcd.setMaxVersions(4); hcd.setEncryptionType(algorithm); hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()),cfKey)); init(name.getMethodName(), conf, hcd, false); this.store.add(new KeyValue(row, family, qf1, 1, value), null); this.store.add(new KeyValue(row, family, qf2, 1, value), null); this.store.add(new KeyValue(row, family, qf3, 1, value), null); flush(1); this.store.add(new KeyValue(row, family, qf4, 1, value), null); this.store.add(new KeyValue(row, family, qf5, 1, value), null); this.store.add(new KeyValue(row, family, qf6, 1, value), null); flush(2); Collection<HStoreFile> storefiles = this.store.getStorefiles(); checkMobHFileEncrytption(storefiles); // Scan the values Scan scan = new Scan(get); InternalScanner scanner = (InternalScanner) store.getScanner(scan, scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0); List<Cell> results = new ArrayList<>(); scanner.next(results); Collections.sort(results, CellComparatorImpl.COMPARATOR); scanner.close(); Assert.assertEquals(expected.size(), results.size()); for(int i=0; i<results.size(); i++) { Assert.assertEquals(expected.get(i), results.get(i)); } // Trigger major compaction this.store.triggerMajorCompaction(); Optional<CompactionContext> requestCompaction = this.store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null); this.store.compact(requestCompaction.get(), NoLimitThroughputController.INSTANCE, null); Assert.assertEquals(1, this.store.getStorefiles().size()); //Check encryption after compaction checkMobHFileEncrytption(this.store.getStorefiles()); }
/** * Apply column family options such as Bloom filters, compression, and data * block encoding. */ protected void applyColumnFamilyOptions(TableName tableName, byte[][] columnFamilies) throws IOException { try (Connection conn = ConnectionFactory.createConnection(conf); Admin admin = conn.getAdmin()) { TableDescriptor tableDesc = admin.getDescriptor(tableName); LOG.info("Disabling table " + tableName); admin.disableTable(tableName); for (byte[] cf : columnFamilies) { ColumnFamilyDescriptor columnDesc = tableDesc.getColumnFamily(cf); boolean isNewCf = columnDesc == null; ColumnFamilyDescriptorBuilder columnDescBuilder = isNewCf ? ColumnFamilyDescriptorBuilder.newBuilder(cf) : ColumnFamilyDescriptorBuilder.newBuilder(columnDesc); if (bloomType != null) { columnDescBuilder.setBloomFilterType(bloomType); } if (compressAlgo != null) { columnDescBuilder.setCompressionType(compressAlgo); } if (dataBlockEncodingAlgo != null) { columnDescBuilder.setDataBlockEncoding(dataBlockEncodingAlgo); } if (inMemoryCF) { columnDescBuilder.setInMemory(inMemoryCF); } if (cipher != null) { byte[] keyBytes = new byte[cipher.getKeyLength()]; new SecureRandom().nextBytes(keyBytes); columnDescBuilder.setEncryptionType(cipher.getName()); columnDescBuilder.setEncryptionKey( EncryptionUtil.wrapKey(conf, User.getCurrent().getShortName(), new SecretKeySpec(keyBytes, cipher.getName()))); } if (mobThreshold >= 0) { columnDescBuilder.setMobEnabled(true); columnDescBuilder.setMobThreshold(mobThreshold); } if (isNewCf) { admin.addColumnFamily(tableName, columnDescBuilder.build()); } else { admin.modifyColumnFamily(tableName, columnDescBuilder.build()); } } LOG.info("Enabling table " + tableName); admin.enableTable(tableName); } }
