@Override protected ErasureCodingStep prepareDecodingStep( final ECBlockGroup blockGroup) { RawErasureDecoder rawDecoder; RawErasureEncoder rawEncoder; ECBlock[] inputBlocks = getInputBlocks(blockGroup); ECBlock[] outputBlocks = getOutputBlocks(blockGroup); rawDecoder = checkCreateRSRawDecoder(); rawEncoder = checkCreateXorRawEncoder(); return new HHXORErasureDecodingStep(inputBlocks, getErasedIndexes(inputBlocks), outputBlocks, rawDecoder, rawEncoder); }
@Test public void testRSDefaultRawCoder() { ErasureCoderOptions coderOptions = new ErasureCoderOptions( numDataUnit, numParityUnit); // should return default raw coder of rs-default codec RawErasureEncoder encoder = CodecUtil.createRawEncoder( conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); Assert.assertTrue(encoder instanceof RSRawEncoder); RawErasureDecoder decoder = CodecUtil.createRawDecoder( conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); Assert.assertTrue(decoder instanceof RSRawDecoder); // should return default raw coder of rs-legacy codec encoder = CodecUtil.createRawEncoder(conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions); Assert.assertTrue(encoder instanceof RSRawEncoderLegacy); decoder = CodecUtil.createRawDecoder(conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions); Assert.assertTrue(decoder instanceof RSRawDecoderLegacy); }
/** * The constructor with all the necessary info. * @param inputBlocks * @param erasedIndexes the indexes of erased blocks in inputBlocks array * @param outputBlocks * @param rawDecoder underlying RS decoder for hitchhiker decoding * @param rawEncoder underlying XOR encoder for hitchhiker decoding */ public HHXORErasureDecodingStep(ECBlock[] inputBlocks, int[] erasedIndexes, ECBlock[] outputBlocks, RawErasureDecoder rawDecoder, RawErasureEncoder rawEncoder) { super(inputBlocks, outputBlocks); this.pbIndex = rawDecoder.getNumParityUnits() - 1; this.erasedIndexes = erasedIndexes; this.rsRawDecoder = rawDecoder; this.xorRawEncoder = rawEncoder; this.piggyBackIndex = HHUtil.initPiggyBackIndexWithoutPBVec( rawDecoder.getNumDataUnits(), rawDecoder.getNumParityUnits()); this.piggyBackFullIndex = HHUtil.initPiggyBackFullIndexVec( rawDecoder.getNumDataUnits(), piggyBackIndex); }
@Override protected ErasureCodingStep prepareDecodingStep( final ECBlockGroup blockGroup) { RawErasureDecoder rawDecoder = CodecUtil.createXORRawDecoder(getConf(), getNumDataUnits(), getNumParityUnits()); ECBlock[] inputBlocks = getInputBlocks(blockGroup); return new ErasureDecodingStep(inputBlocks, getErasedIndexes(inputBlocks), getOutputBlocks(blockGroup), rawDecoder); }
/** * The constructor with all the necessary info. * @param inputBlocks * @param erasedIndexes the indexes of erased blocks in inputBlocks array * @param outputBlocks * @param rawDecoder */ public ErasureDecodingStep(ECBlock[] inputBlocks, int[] erasedIndexes, ECBlock[] outputBlocks, RawErasureDecoder rawDecoder) { super(inputBlocks, outputBlocks); this.erasedIndexes = erasedIndexes; this.rawDecoder = rawDecoder; }
@Override protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) { ECBlock[] inputBlocks = getInputBlocks(blockGroup); ECBlock[] outputBlocks = getOutputBlocks(blockGroup); RawErasureDecoder rawDecoder = checkCreateRSRawDecoder(); return new ErasureDecodingStep(inputBlocks, getErasedIndexes(inputBlocks), outputBlocks, rawDecoder); }
private RawErasureDecoder checkCreateRSRawDecoder() { if (rsRawDecoder == null) { rsRawDecoder = CodecUtil.createRSRawDecoder(getConf(), getNumDataUnits(), getNumParityUnits()); } return rsRawDecoder; }
/** * Create RS raw decoder according to configuration. * @param conf configuration possibly with some items to configure the coder * @param numDataUnits number of data units in a coding group * @param numParityUnits number of parity units in a coding group * @return raw decoder */ public static RawErasureDecoder createRSRawDecoder( Configuration conf, int numDataUnits, int numParityUnits) { RawErasureCoder rawCoder = createRawCoder(conf, CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, false, numDataUnits, numParityUnits); if (rawCoder == null) { rawCoder = new RSRawDecoder(numDataUnits, numParityUnits); } return (RawErasureDecoder) rawCoder; }
/** * Create XOR raw decoder according to configuration. * @param conf configuration possibly with some items to configure the coder * @param numDataUnits number of data units in a coding group * @param numParityUnits number of parity units in a coding group * @return raw decoder */ public static RawErasureDecoder createXORRawDecoder( Configuration conf, int numDataUnits, int numParityUnits) { RawErasureCoder rawCoder = createRawCoder(conf, CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY, false, numDataUnits, numParityUnits); if (rawCoder == null) { rawCoder = new XORRawDecoder(numDataUnits, numParityUnits); } return (RawErasureDecoder) rawCoder; }
/** * Decode based on the given input buffers and erasure coding policy. */ public static void decodeAndFillBuffer(final byte[][] decodeInputs, AlignedStripe alignedStripe, int dataBlkNum, int parityBlkNum, RawErasureDecoder decoder) { // Step 1: prepare indices and output buffers for missing data units int[] decodeIndices = new int[parityBlkNum]; int pos = 0; for (int i = 0; i < dataBlkNum; i++) { if (alignedStripe.chunks[i] != null && alignedStripe.chunks[i].state == StripingChunk.MISSING){ decodeIndices[pos++] = i; } } decodeIndices = Arrays.copyOf(decodeIndices, pos); byte[][] decodeOutputs = new byte[decodeIndices.length][(int) alignedStripe.getSpanInBlock()]; // Step 2: decode into prepared output buffers decoder.decode(decodeInputs, decodeIndices, decodeOutputs); // Step 3: fill original application buffer with decoded data for (int i = 0; i < decodeIndices.length; i++) { int missingBlkIdx = decodeIndices[i]; StripingChunk chunk = alignedStripe.chunks[missingBlkIdx]; if (chunk.state == StripingChunk.MISSING) { chunk.copyFrom(decodeOutputs[i]); } } }
private RawErasureDecoder checkCreateRSRawDecoder() { if (rsRawDecoder == null) { ErasureCoderOptions coderOptions = new ErasureCoderOptions( getNumDataUnits(), getNumParityUnits()); rsRawDecoder = CodecUtil.createRawDecoder(getConf(), ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); } return rsRawDecoder; }
@Override protected ErasureCodingStep prepareDecodingStep( final ECBlockGroup blockGroup) { ErasureCoderOptions coderOptions = new ErasureCoderOptions( getNumDataUnits(), getNumParityUnits()); RawErasureDecoder rawDecoder = CodecUtil.createRawDecoder(getConf(), ErasureCodeConstants.XOR_CODEC_NAME, coderOptions); ECBlock[] inputBlocks = getInputBlocks(blockGroup); return new ErasureDecodingStep(inputBlocks, getErasedIndexes(inputBlocks), getOutputBlocks(blockGroup), rawDecoder); }
private RawErasureDecoder checkCreateRSRawDecoder() { if (rsRawDecoder == null) { // TODO: we should create the raw coder according to codec. ErasureCoderOptions coderOptions = new ErasureCoderOptions( getNumDataUnits(), getNumParityUnits()); rsRawDecoder = CodecUtil.createRawDecoder(getConf(), ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); } return rsRawDecoder; }
/** * Create RS raw decoder according to configuration. * @param conf configuration * @param coderOptions coder options that's used to create the coder * @param codec the codec to use. If null, will use the default codec * @return raw decoder */ public static RawErasureDecoder createRawDecoder( Configuration conf, String codec, ErasureCoderOptions coderOptions) { Preconditions.checkNotNull(conf); Preconditions.checkNotNull(codec); String rawCoderFactoryKey = getFactNameFromCodec(conf, codec); RawErasureCoderFactory fact = createRawCoderFactory(conf, rawCoderFactoryKey); return fact.createDecoder(coderOptions); }
private RawErasureDecoder newDecoder(int numDataUnits, int numParityUnits) { return CodecUtil.createRSRawDecoder(conf, numDataUnits, numParityUnits); }
