private WALEntry createEntry(TableName table, int row, KeyValue.Type type, List<Cell> cells) { byte[] fam = table.equals(TABLE_NAME1) ? FAM_NAME1 : FAM_NAME2; byte[] rowBytes = Bytes.toBytes(row); // Just make sure we don't get the same ts for two consecutive rows with // same key try { Thread.sleep(1); } catch (InterruptedException e) { LOG.info("Was interrupted while sleep, meh", e); } final long now = System.currentTimeMillis(); KeyValue kv = null; if(type.getCode() == KeyValue.Type.Put.getCode()) { kv = new KeyValue(rowBytes, fam, fam, now, KeyValue.Type.Put, Bytes.toBytes(row)); } else if (type.getCode() == KeyValue.Type.DeleteColumn.getCode()) { kv = new KeyValue(rowBytes, fam, fam, now, KeyValue.Type.DeleteColumn); } else if (type.getCode() == KeyValue.Type.DeleteFamily.getCode()) { kv = new KeyValue(rowBytes, fam, null, now, KeyValue.Type.DeleteFamily); } WALEntry.Builder builder = WALEntry.newBuilder(); builder.setAssociatedCellCount(1); WALKey.Builder keyBuilder = WALKey.newBuilder(); UUID.Builder uuidBuilder = UUID.newBuilder(); uuidBuilder.setLeastSigBits(HConstants.DEFAULT_CLUSTER_ID.getLeastSignificantBits()); uuidBuilder.setMostSigBits(HConstants.DEFAULT_CLUSTER_ID.getMostSignificantBits()); keyBuilder.setClusterId(uuidBuilder.build()); keyBuilder.setTableName(ByteStringer.wrap(table.getName())); keyBuilder.setWriteTime(now); keyBuilder.setEncodedRegionName(ByteStringer.wrap(HConstants.EMPTY_BYTE_ARRAY)); keyBuilder.setLogSequenceNumber(-1); builder.setKey(keyBuilder.build()); cells.add(kv); return builder.build(); }
public WALKey.Builder getBuilder( WALCellCodec.ByteStringCompressor compressor) throws IOException { WALKey.Builder builder = WALKey.newBuilder(); if (compressionContext == null) { builder.setEncodedRegionName(HBaseZeroCopyByteString.wrap(this.encodedRegionName)); builder.setTableName(HBaseZeroCopyByteString.wrap(this.tablename.getName())); } else { builder.setEncodedRegionName( compressor.compress(this.encodedRegionName, compressionContext.regionDict)); builder.setTableName(compressor.compress(this.tablename.getName(), compressionContext.tableDict)); } builder.setLogSequenceNumber(this.logSeqNum); builder.setWriteTime(writeTime); if (this.nonce != HConstants.NO_NONCE) { builder.setNonce(nonce); } if (this.nonceGroup != HConstants.NO_NONCE) { builder.setNonceGroup(nonceGroup); } HBaseProtos.UUID.Builder uuidBuilder = HBaseProtos.UUID.newBuilder(); for (UUID clusterId : clusterIds) { uuidBuilder.setLeastSigBits(clusterId.getLeastSignificantBits()); uuidBuilder.setMostSigBits(clusterId.getMostSignificantBits()); builder.addClusterIds(uuidBuilder.build()); } if (scopes != null) { for (Map.Entry<byte[], Integer> e : scopes.entrySet()) { ByteString family = (compressionContext == null) ? HBaseZeroCopyByteString.wrap(e.getKey()) : compressor.compress(e.getKey(), compressionContext.familyDict); builder.addScopes(FamilyScope.newBuilder() .setFamily(family).setScopeType(ScopeType.valueOf(e.getValue()))); } } return builder; }
public void readFieldsFromPb( WALKey walKey, WALCellCodec.ByteStringUncompressor uncompressor) throws IOException { if (this.compressionContext != null) { this.encodedRegionName = uncompressor.uncompress( walKey.getEncodedRegionName(), compressionContext.regionDict); byte[] tablenameBytes = uncompressor.uncompress( walKey.getTableName(), compressionContext.tableDict); this.tablename = TableName.valueOf(tablenameBytes); } else { this.encodedRegionName = walKey.getEncodedRegionName().toByteArray(); this.tablename = TableName.valueOf(walKey.getTableName().toByteArray()); } clusterIds.clear(); if (walKey.hasClusterId()) { //When we are reading the older log (0.95.1 release) //This is definitely the originating cluster clusterIds.add(new UUID(walKey.getClusterId().getMostSigBits(), walKey.getClusterId() .getLeastSigBits())); } for (HBaseProtos.UUID clusterId : walKey.getClusterIdsList()) { clusterIds.add(new UUID(clusterId.getMostSigBits(), clusterId.getLeastSigBits())); } if (walKey.hasNonceGroup()) { this.nonceGroup = walKey.getNonceGroup(); } if (walKey.hasNonce()) { this.nonce = walKey.getNonce(); } this.scopes = null; if (walKey.getScopesCount() > 0) { this.scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR); for (FamilyScope scope : walKey.getScopesList()) { byte[] family = (compressionContext == null) ? scope.getFamily().toByteArray() : uncompressor.uncompress(scope.getFamily(), compressionContext.familyDict); this.scopes.put(family, scope.getScopeType().getNumber()); } } this.logSeqNum = walKey.getLogSequenceNumber(); this.writeTime = walKey.getWriteTime(); }
private WALEntry createEntry(byte [] table, int row, KeyValue.Type type, List<Cell> cells) { byte[] fam = Bytes.equals(table, TABLE_NAME1) ? FAM_NAME1 : FAM_NAME2; byte[] rowBytes = Bytes.toBytes(row); // Just make sure we don't get the same ts for two consecutive rows with // same key try { Thread.sleep(1); } catch (InterruptedException e) { LOG.info("Was interrupted while sleep, meh", e); } final long now = System.currentTimeMillis(); KeyValue kv = null; if(type.getCode() == KeyValue.Type.Put.getCode()) { kv = new KeyValue(rowBytes, fam, fam, now, KeyValue.Type.Put, Bytes.toBytes(row)); } else if (type.getCode() == KeyValue.Type.DeleteColumn.getCode()) { kv = new KeyValue(rowBytes, fam, fam, now, KeyValue.Type.DeleteColumn); } else if (type.getCode() == KeyValue.Type.DeleteFamily.getCode()) { kv = new KeyValue(rowBytes, fam, null, now, KeyValue.Type.DeleteFamily); } WALEntry.Builder builder = WALEntry.newBuilder(); builder.setAssociatedCellCount(1); WALKey.Builder keyBuilder = WALKey.newBuilder(); UUID.Builder uuidBuilder = UUID.newBuilder(); uuidBuilder.setLeastSigBits(HConstants.DEFAULT_CLUSTER_ID.getLeastSignificantBits()); uuidBuilder.setMostSigBits(HConstants.DEFAULT_CLUSTER_ID.getMostSignificantBits()); keyBuilder.setClusterId(uuidBuilder.build()); keyBuilder.setTableName(HBaseZeroCopyByteString.wrap(table)); keyBuilder.setWriteTime(now); keyBuilder.setEncodedRegionName(HBaseZeroCopyByteString.wrap(HConstants.EMPTY_BYTE_ARRAY)); keyBuilder.setLogSequenceNumber(-1); builder.setKey(keyBuilder.build()); cells.add(kv); return builder.build(); }
public WALKey.Builder getBuilder(WALCellCodec.ByteStringCompressor compressor) throws IOException { WALKey.Builder builder = WALKey.newBuilder(); if (compressionContext == null) { builder.setEncodedRegionName(HBaseZeroCopyByteString.wrap(this.encodedRegionName)); builder.setTableName(HBaseZeroCopyByteString.wrap(this.tablename.getName())); } else { builder.setEncodedRegionName(compressor.compress(this.encodedRegionName, compressionContext.regionDict)); builder.setTableName(compressor.compress(this.tablename.getName(), compressionContext.tableDict)); } builder.setLogSequenceNumber(this.logSeqNum); builder.setWriteTime(writeTime); if (this.nonce != HConstants.NO_NONCE) { builder.setNonce(nonce); } if (this.nonceGroup != HConstants.NO_NONCE) { builder.setNonceGroup(nonceGroup); } HBaseProtos.UUID.Builder uuidBuilder = HBaseProtos.UUID.newBuilder(); for (UUID clusterId : clusterIds) { uuidBuilder.setLeastSigBits(clusterId.getLeastSignificantBits()); uuidBuilder.setMostSigBits(clusterId.getMostSignificantBits()); builder.addClusterIds(uuidBuilder.build()); } if (scopes != null) { for (Map.Entry<byte[], Integer> e : scopes.entrySet()) { ByteString family = (compressionContext == null) ? HBaseZeroCopyByteString.wrap(e.getKey()) : compressor.compress(e.getKey(), compressionContext.familyDict); builder.addScopes(FamilyScope.newBuilder() .setFamily(family).setScopeType(ScopeType.valueOf(e.getValue()))); } } return builder; }
public WALKey.Builder getBuilder( WALCellCodec.ByteStringCompressor compressor) throws IOException { WALKey.Builder builder = WALKey.newBuilder(); if (compressionContext == null) { builder.setEncodedRegionName(ZeroCopyLiteralByteString.wrap(this.encodedRegionName)); builder.setTableName(ZeroCopyLiteralByteString.wrap(this.tablename.getName())); } else { builder.setEncodedRegionName( compressor.compress(this.encodedRegionName, compressionContext.regionDict)); builder.setTableName(compressor.compress(this.tablename.getName(), compressionContext.tableDict)); } builder.setLogSequenceNumber(this.logSeqNum); builder.setWriteTime(writeTime); HBaseProtos.UUID.Builder uuidBuilder = HBaseProtos.UUID.newBuilder(); for (UUID clusterId : clusterIds) { uuidBuilder.setLeastSigBits(clusterId.getLeastSignificantBits()); uuidBuilder.setMostSigBits(clusterId.getMostSignificantBits()); builder.addClusterIds(uuidBuilder.build()); } if (scopes != null) { for (Map.Entry<byte[], Integer> e : scopes.entrySet()) { ByteString family = (compressionContext == null) ? ZeroCopyLiteralByteString.wrap(e.getKey()) : compressor.compress(e.getKey(), compressionContext.familyDict); builder.addScopes(FamilyScope.newBuilder() .setFamily(family).setScopeType(ScopeType.valueOf(e.getValue()))); } } return builder; }
public void readFieldsFromPb( WALKey walKey, WALCellCodec.ByteStringUncompressor uncompressor) throws IOException { if (this.compressionContext != null) { this.encodedRegionName = uncompressor.uncompress( walKey.getEncodedRegionName(), compressionContext.regionDict); byte[] tablenameBytes = uncompressor.uncompress( walKey.getTableName(), compressionContext.tableDict); this.tablename = TableName.valueOf(tablenameBytes); } else { this.encodedRegionName = walKey.getEncodedRegionName().toByteArray(); this.tablename = TableName.valueOf(walKey.getTableName().toByteArray()); } clusterIds.clear(); if (walKey.hasClusterId()) { //When we are reading the older log (0.95.1 release) //This is definitely the originating cluster clusterIds.add(new UUID(walKey.getClusterId().getMostSigBits(), walKey.getClusterId() .getLeastSigBits())); } for (HBaseProtos.UUID clusterId : walKey.getClusterIdsList()) { clusterIds.add(new UUID(clusterId.getMostSigBits(), clusterId.getLeastSigBits())); } this.scopes = null; if (walKey.getScopesCount() > 0) { this.scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR); for (FamilyScope scope : walKey.getScopesList()) { byte[] family = (compressionContext == null) ? scope.getFamily().toByteArray() : uncompressor.uncompress(scope.getFamily(), compressionContext.familyDict); this.scopes.put(family, scope.getScopeType().getNumber()); } } this.logSeqNum = walKey.getLogSequenceNumber(); this.writeTime = walKey.getWriteTime(); }
private WALEntry createEntry(byte [] table, int row, KeyValue.Type type, List<Cell> cells) { byte[] fam = Bytes.equals(table, TABLE_NAME1) ? FAM_NAME1 : FAM_NAME2; byte[] rowBytes = Bytes.toBytes(row); // Just make sure we don't get the same ts for two consecutive rows with // same key try { Thread.sleep(1); } catch (InterruptedException e) { LOG.info("Was interrupted while sleep, meh", e); } final long now = System.currentTimeMillis(); KeyValue kv = null; if(type.getCode() == KeyValue.Type.Put.getCode()) { kv = new KeyValue(rowBytes, fam, fam, now, KeyValue.Type.Put, Bytes.toBytes(row)); } else if (type.getCode() == KeyValue.Type.DeleteColumn.getCode()) { kv = new KeyValue(rowBytes, fam, fam, now, KeyValue.Type.DeleteColumn); } else if (type.getCode() == KeyValue.Type.DeleteFamily.getCode()) { kv = new KeyValue(rowBytes, fam, null, now, KeyValue.Type.DeleteFamily); } WALEntry.Builder builder = WALEntry.newBuilder(); builder.setAssociatedCellCount(1); WALKey.Builder keyBuilder = WALKey.newBuilder(); UUID.Builder uuidBuilder = UUID.newBuilder(); uuidBuilder.setLeastSigBits(HConstants.DEFAULT_CLUSTER_ID.getLeastSignificantBits()); uuidBuilder.setMostSigBits(HConstants.DEFAULT_CLUSTER_ID.getMostSignificantBits()); keyBuilder.setClusterId(uuidBuilder.build()); keyBuilder.setTableName(ZeroCopyLiteralByteString.wrap(table)); keyBuilder.setWriteTime(now); keyBuilder.setEncodedRegionName(ZeroCopyLiteralByteString.wrap(HConstants.EMPTY_BYTE_ARRAY)); keyBuilder.setLogSequenceNumber(-1); builder.setKey(keyBuilder.build()); cells.add(kv); return builder.build(); }
/** * This function is used to construct mutations from a WALEntry. It also reconstructs HLogKey & * WALEdit from the passed in WALEntry * @param entry * @param cells * @param logEntry pair of HLogKey and WALEdit instance stores HLogKey and WALEdit instances * extracted from the passed in WALEntry. * @return list of Pair<MutationType, Mutation> to be replayed * @throws IOException */ public static List<Pair<MutationType, Mutation>> getMutationsFromWALEntry(WALEntry entry, CellScanner cells, Pair<HLogKey, WALEdit> logEntry) throws IOException { if (entry == null) { // return an empty array return new ArrayList<Pair<MutationType, Mutation>>(); } int count = entry.getAssociatedCellCount(); List<Pair<MutationType, Mutation>> mutations = new ArrayList<Pair<MutationType, Mutation>>(); Cell previousCell = null; Mutation m = null; HLogKey key = null; WALEdit val = null; if (logEntry != null) val = new WALEdit(); for (int i = 0; i < count; i++) { // Throw index out of bounds if our cell count is off if (!cells.advance()) { throw new ArrayIndexOutOfBoundsException("Expected=" + count + ", index=" + i); } Cell cell = cells.current(); if (val != null) val.add(KeyValueUtil.ensureKeyValue(cell)); boolean isNewRowOrType = previousCell == null || previousCell.getTypeByte() != cell.getTypeByte() || !CellUtil.matchingRow(previousCell, cell); if (isNewRowOrType) { // Create new mutation if (CellUtil.isDelete(cell)) { m = new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); mutations.add(new Pair<MutationType, Mutation>(MutationType.DELETE, m)); } else { m = new Put(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); mutations.add(new Pair<MutationType, Mutation>(MutationType.PUT, m)); } } if (CellUtil.isDelete(cell)) { ((Delete) m).addDeleteMarker(KeyValueUtil.ensureKeyValue(cell)); } else { ((Put) m).add(KeyValueUtil.ensureKeyValue(cell)); } previousCell = cell; } // reconstruct HLogKey if (logEntry != null) { WALKey walKey = entry.getKey(); List<UUID> clusterIds = new ArrayList<UUID>(walKey.getClusterIdsCount()); for (HBaseProtos.UUID uuid : entry.getKey().getClusterIdsList()) { clusterIds.add(new UUID(uuid.getMostSigBits(), uuid.getLeastSigBits())); } key = new HLogKey(walKey.getEncodedRegionName().toByteArray(), TableName.valueOf(walKey .getTableName().toByteArray()), walKey.getLogSequenceNumber(), walKey.getWriteTime(), clusterIds); logEntry.setFirst(key); logEntry.setSecond(val); } return mutations; }