Java 类org.apache.hadoop.io.erasurecode.ECBlockGroup 实例源码

@Override
protected ErasureCodingStep prepareDecodingStep(
        final ECBlockGroup blockGroup) {

  RawErasureDecoder rawDecoder;
  RawErasureEncoder rawEncoder;

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);
  ECBlock[] outputBlocks = getOutputBlocks(blockGroup);

  rawDecoder = checkCreateRSRawDecoder();
  rawEncoder = checkCreateXorRawEncoder();

  return new HHXORErasureDecodingStep(inputBlocks,
          getErasedIndexes(inputBlocks), outputBlocks, rawDecoder,
          rawEncoder);
}

/**
 * Which blocks were erased ?
 * @param blockGroup
 * @return output blocks to recover
 */
protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
  ECBlock[] outputBlocks = new ECBlock[getNumErasedBlocks(blockGroup)];

  int idx = 0;

  for (int i = 0; i < getNumDataUnits(); i++) {
    if (blockGroup.getDataBlocks()[i].isErased()) {
      outputBlocks[idx++] = blockGroup.getDataBlocks()[i];
    }
  }

  for (int i = 0; i < getNumParityUnits(); i++) {
    if (blockGroup.getParityBlocks()[i].isErased()) {
      outputBlocks[idx++] = blockGroup.getParityBlocks()[i];
    }
  }

  return outputBlocks;
}

/**
 * Which blocks were erased ?
 * @param blockGroup
 * @return output blocks to recover
 */
protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
  ECBlock[] outputBlocks = new ECBlock[getNumErasedBlocks(blockGroup)];

  int idx = 0;

  for (int i = 0; i < getNumDataUnits(); i++) {
    if (blockGroup.getDataBlocks()[i].isErased()) {
      outputBlocks[idx++] = blockGroup.getDataBlocks()[i];
    }
  }

  for (int i = 0; i < getNumParityUnits(); i++) {
    if (blockGroup.getParityBlocks()[i].isErased()) {
      outputBlocks[idx++] = blockGroup.getParityBlocks()[i];
    }
  }

  return outputBlocks;
}

@Override
protected ErasureCodingStep prepareDecodingStep(
        final ECBlockGroup blockGroup) {

  RawErasureDecoder rawDecoder;
  RawErasureEncoder rawEncoder;

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);
  ECBlock[] outputBlocks = getOutputBlocks(blockGroup);

  rawDecoder = checkCreateRSRawDecoder();
  rawEncoder = checkCreateXorRawEncoder();

  return new HHXORErasureDecodingStep(inputBlocks,
          getErasedIndexes(inputBlocks), outputBlocks, rawDecoder,
          rawEncoder);
}

/**
 * Which blocks were erased ?
 * @param blockGroup
 * @return output blocks to recover
 */
protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
  ECBlock[] outputBlocks = new ECBlock[getNumErasedBlocks(blockGroup)];

  int idx = 0;

  for (int i = 0; i < getNumDataUnits(); i++) {
    if (blockGroup.getDataBlocks()[i].isErased()) {
      outputBlocks[idx++] = blockGroup.getDataBlocks()[i];
    }
  }

  for (int i = 0; i < getNumParityUnits(); i++) {
    if (blockGroup.getParityBlocks()[i].isErased()) {
      outputBlocks[idx++] = blockGroup.getParityBlocks()[i];
    }
  }

  return outputBlocks;
}

@Override
protected ErasureCodingStep prepareEncodingStep(
        final ECBlockGroup blockGroup) {

  RawErasureEncoder rsRawEncoderTmp = checkCreateRSRawEncoder();
  RawErasureEncoder xorRawEncoderTmp = checkCreateXorRawEncoder();

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new HHXORErasureEncodingStep(inputBlocks,
          getOutputBlocks(blockGroup), rsRawEncoderTmp, xorRawEncoderTmp);
}

@Override
protected ErasureCodingStep prepareDecodingStep(
    final ECBlockGroup blockGroup) {
  RawErasureDecoder rawDecoder = CodecUtil.createXORRawDecoder(getConf(),
      getNumDataUnits(), getNumParityUnits());

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new ErasureDecodingStep(inputBlocks,
      getErasedIndexes(inputBlocks),
      getOutputBlocks(blockGroup), rawDecoder);
}

/**
 * Which blocks were erased ? For XOR it's simple we only allow and return one
 * erased block, either data or parity.
 * @param blockGroup
 * @return output blocks to recover
 */
@Override
protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
  /**
   * If more than one blocks (either data or parity) erased, then it's not
   * edible to recover. We don't have the check here since it will be done
   * by upper level: ErasreCoder call can be avoid if not possible to recover
   * at all.
   */
  int erasedNum = getNumErasedBlocks(blockGroup);
  ECBlock[] outputBlocks = new ECBlock[erasedNum];

  int idx = 0;
  for (int i = 0; i < getNumParityUnits(); i++) {
    if (blockGroup.getParityBlocks()[i].isErased()) {
      outputBlocks[idx++] = blockGroup.getParityBlocks()[i];
    }
  }

  for (int i = 0; i < getNumDataUnits(); i++) {
    if (blockGroup.getDataBlocks()[i].isErased()) {
      outputBlocks[idx++] = blockGroup.getDataBlocks()[i];
    }
  }

  return outputBlocks;
}

@Override
protected ErasureCodingStep prepareEncodingStep(
    final ECBlockGroup blockGroup) {
  RawErasureEncoder rawEncoder = CodecUtil.createXORRawEncoder(getConf(),
      getNumDataUnits(), getNumParityUnits());

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new ErasureEncodingStep(inputBlocks,
      getOutputBlocks(blockGroup), rawEncoder);
}

/**
 * We have all the data blocks and parity blocks as input blocks for
 * recovering by default. It's codec specific
 * @param blockGroup
 * @return input blocks
 */
protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
  ECBlock[] inputBlocks = new ECBlock[getNumDataUnits() +
          getNumParityUnits()];

  System.arraycopy(blockGroup.getDataBlocks(), 0, inputBlocks,
          0, getNumDataUnits());

  System.arraycopy(blockGroup.getParityBlocks(), 0, inputBlocks,
          getNumDataUnits(), getNumParityUnits());

  return inputBlocks;
}

@Override
protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) {

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);
  ECBlock[] outputBlocks = getOutputBlocks(blockGroup);

  RawErasureDecoder rawDecoder = checkCreateRSRawDecoder();
  return new ErasureDecodingStep(inputBlocks,
      getErasedIndexes(inputBlocks), outputBlocks, rawDecoder);
}

/**
 * Calculating and organizing BlockGroup, to be called by ECManager
 * @param dataBlocks Data blocks to compute parity blocks against
 * @param parityBlocks To be computed parity blocks
 * @return
 */
public ECBlockGroup makeBlockGroup(ECBlock[] dataBlocks,
                                   ECBlock[] parityBlocks) {

  ECBlockGroup blockGroup = new ECBlockGroup(dataBlocks, parityBlocks);
  return blockGroup;
}

/**
 * Prepare a block group for encoding.
 * @return
 */
protected ECBlockGroup prepareBlockGroupForEncoding() {
  ECBlock[] dataBlocks = new TestBlock[numDataUnits];
  ECBlock[] parityBlocks = new TestBlock[numParityUnits];

  for (int i = 0; i < numDataUnits; i++) {
    dataBlocks[i] = generateDataBlock();
  }

  for (int i = 0; i < numParityUnits; i++) {
    parityBlocks[i] = allocateOutputBlock();
  }

  return new ECBlockGroup(dataBlocks, parityBlocks);
}

@Override
protected ErasureCodingStep prepareDecodingStep(
    final ECBlockGroup blockGroup) {
  RawErasureDecoder rawDecoder = CodecUtil.createXORRawDecoder(getConf(),
      getNumDataUnits(), getNumParityUnits());

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new ErasureDecodingStep(inputBlocks,
      getErasedIndexes(inputBlocks),
      getOutputBlocks(blockGroup), rawDecoder);
}

/**
 * Which blocks were erased ? For XOR it's simple we only allow and return one
 * erased block, either data or parity.
 * @param blockGroup
 * @return output blocks to recover
 */
@Override
protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
  /**
   * If more than one blocks (either data or parity) erased, then it's not
   * edible to recover. We don't have the check here since it will be done
   * by upper level: ErasreCoder call can be avoid if not possible to recover
   * at all.
   */
  int erasedNum = getNumErasedBlocks(blockGroup);
  ECBlock[] outputBlocks = new ECBlock[erasedNum];

  int idx = 0;
  for (int i = 0; i < getNumParityUnits(); i++) {
    if (blockGroup.getParityBlocks()[i].isErased()) {
      outputBlocks[idx++] = blockGroup.getParityBlocks()[i];
    }
  }

  for (int i = 0; i < getNumDataUnits(); i++) {
    if (blockGroup.getDataBlocks()[i].isErased()) {
      outputBlocks[idx++] = blockGroup.getDataBlocks()[i];
    }
  }

  return outputBlocks;
}

@Override
protected ErasureCodingStep prepareEncodingStep(
    final ECBlockGroup blockGroup) {
  RawErasureEncoder rawEncoder = CodecUtil.createXORRawEncoder(getConf(),
      getNumDataUnits(), getNumParityUnits());

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new ErasureEncodingStep(inputBlocks,
      getOutputBlocks(blockGroup), rawEncoder);
}

/**
 * We have all the data blocks and parity blocks as input blocks for
 * recovering by default. It's codec specific
 * @param blockGroup
 * @return
 */
protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
  ECBlock[] inputBlocks = new ECBlock[getNumDataUnits() +
          getNumParityUnits()];

  System.arraycopy(blockGroup.getDataBlocks(), 0, inputBlocks,
          0, getNumDataUnits());

  System.arraycopy(blockGroup.getParityBlocks(), 0, inputBlocks,
          getNumDataUnits(), getNumParityUnits());

  return inputBlocks;
}

@Override
protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) {

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);
  ECBlock[] outputBlocks = getOutputBlocks(blockGroup);

  RawErasureDecoder rawDecoder = checkCreateRSRawDecoder();
  return new ErasureDecodingStep(inputBlocks,
      getErasedIndexes(inputBlocks), outputBlocks, rawDecoder);
}

/**
 * Calculating and organizing BlockGroup, to be called by ECManager
 * @param dataBlocks Data blocks to compute parity blocks against
 * @param parityBlocks To be computed parity blocks
 * @return
 */
public ECBlockGroup makeBlockGroup(ECBlock[] dataBlocks,
                                   ECBlock[] parityBlocks) {

  ECBlockGroup blockGroup = new ECBlockGroup(dataBlocks, parityBlocks);
  return blockGroup;
}

/**
 * Prepare a block group for encoding.
 * @return
 */
protected ECBlockGroup prepareBlockGroupForEncoding() {
  ECBlock[] dataBlocks = new TestBlock[numDataUnits];
  ECBlock[] parityBlocks = new TestBlock[numParityUnits];

  for (int i = 0; i < numDataUnits; i++) {
    dataBlocks[i] = generateDataBlock();
  }

  for (int i = 0; i < numParityUnits; i++) {
    parityBlocks[i] = allocateOutputBlock();
  }

  return new ECBlockGroup(dataBlocks, parityBlocks);
}

@Override
protected ErasureCodingStep prepareEncodingStep(
        final ECBlockGroup blockGroup) {

  RawErasureEncoder rsRawEncoderTmp = checkCreateRSRawEncoder();
  RawErasureEncoder xorRawEncoderTmp = checkCreateXorRawEncoder();

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new HHXORErasureEncodingStep(inputBlocks,
          getOutputBlocks(blockGroup), rsRawEncoderTmp, xorRawEncoderTmp);
}

@Override
protected ErasureCodingStep prepareDecodingStep(
    final ECBlockGroup blockGroup) {
  ErasureCoderOptions coderOptions = new ErasureCoderOptions(
      getNumDataUnits(), getNumParityUnits());
  RawErasureDecoder rawDecoder = CodecUtil.createRawDecoder(getConf(),
      ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new ErasureDecodingStep(inputBlocks,
      getErasedIndexes(inputBlocks),
      getOutputBlocks(blockGroup), rawDecoder);
}

/**
 * Which blocks were erased ? For XOR it's simple we only allow and return one
 * erased block, either data or parity.
 * @param blockGroup
 * @return output blocks to recover
 */
@Override
protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
  /**
   * If more than one blocks (either data or parity) erased, then it's not
   * edible to recover. We don't have the check here since it will be done
   * by upper level: ErasreCoder call can be avoid if not possible to recover
   * at all.
   */
  int erasedNum = getNumErasedBlocks(blockGroup);
  ECBlock[] outputBlocks = new ECBlock[erasedNum];

  int idx = 0;
  for (int i = 0; i < getNumParityUnits(); i++) {
    if (blockGroup.getParityBlocks()[i].isErased()) {
      outputBlocks[idx++] = blockGroup.getParityBlocks()[i];
    }
  }

  for (int i = 0; i < getNumDataUnits(); i++) {
    if (blockGroup.getDataBlocks()[i].isErased()) {
      outputBlocks[idx++] = blockGroup.getDataBlocks()[i];
    }
  }

  return outputBlocks;
}

@Override
protected ErasureCodingStep prepareEncodingStep(
    final ECBlockGroup blockGroup) {
  ErasureCoderOptions coderOptions = new ErasureCoderOptions(
      getNumDataUnits(), getNumParityUnits());
  RawErasureEncoder rawEncoder = CodecUtil.createRawEncoder(getConf(),
      ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new ErasureEncodingStep(inputBlocks,
      getOutputBlocks(blockGroup), rawEncoder);
}

/**
 * We have all the data blocks and parity blocks as input blocks for
 * recovering by default. It's codec specific
 * @param blockGroup
 * @return input blocks
 */
protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
  ECBlock[] inputBlocks = new ECBlock[getNumDataUnits() +
          getNumParityUnits()];

  System.arraycopy(blockGroup.getDataBlocks(), 0, inputBlocks,
          0, getNumDataUnits());

  System.arraycopy(blockGroup.getParityBlocks(), 0, inputBlocks,
          getNumDataUnits(), getNumParityUnits());

  return inputBlocks;
}

@Override
protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) {

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);
  ECBlock[] outputBlocks = getOutputBlocks(blockGroup);

  RawErasureDecoder rawDecoder = checkCreateRSRawDecoder();
  return new ErasureDecodingStep(inputBlocks,
      getErasedIndexes(inputBlocks), outputBlocks, rawDecoder);
}

/**
 * Calculating and organizing BlockGroup, to be called by ECManager
 * @param dataBlocks Data blocks to compute parity blocks against
 * @param parityBlocks To be computed parity blocks
 * @return
 */
public ECBlockGroup makeBlockGroup(ECBlock[] dataBlocks,
                                   ECBlock[] parityBlocks) {

  ECBlockGroup blockGroup = new ECBlockGroup(dataBlocks, parityBlocks);
  return blockGroup;
}

/**
 * Prepare a block group for encoding.
 * @return
 */
protected ECBlockGroup prepareBlockGroupForEncoding() {
  ECBlock[] dataBlocks = new TestBlock[numDataUnits];
  ECBlock[] parityBlocks = new TestBlock[numParityUnits];

  for (int i = 0; i < numDataUnits; i++) {
    dataBlocks[i] = generateDataBlock();
  }

  for (int i = 0; i < numParityUnits; i++) {
    parityBlocks[i] = allocateOutputBlock();
  }

  return new ECBlockGroup(dataBlocks, parityBlocks);
}

@Override
public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
  // We may have more than this when considering complicate cases. HADOOP-11550
  return prepareEncodingStep(blockGroup);
}

protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
  return blockGroup.getDataBlocks();
}

protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
  return blockGroup.getParityBlocks();
}

@Override
public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
  // We may have more than this when considering complicate cases. HADOOP-11550
  return prepareDecodingStep(blockGroup);
}

/**
 * Get the number of erased blocks in the block group.
 * @param blockGroup
 * @return number of erased blocks
 */
protected int getNumErasedBlocks(ECBlockGroup blockGroup) {
  int num = getNumErasedBlocks(blockGroup.getParityBlocks());
  num += getNumErasedBlocks(blockGroup.getDataBlocks());
  return num;
}

@Override
public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
  // We may have more than this when considering complicate cases. HADOOP-11550
  return prepareEncodingStep(blockGroup);
}

protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
  return blockGroup.getDataBlocks();
}

protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
  return blockGroup.getParityBlocks();
}

@Override
public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
  // We may have more than this when considering complicate cases. HADOOP-11550
  return prepareDecodingStep(blockGroup);
}

/**
 * Get the number of erased blocks in the block group.
 * @param blockGroup
 * @return number of erased blocks
 */
protected int getNumErasedBlocks(ECBlockGroup blockGroup) {
  int num = getNumErasedBlocks(blockGroup.getParityBlocks());
  num += getNumErasedBlocks(blockGroup.getDataBlocks());
  return num;
}

@Override
public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
  // We may have more than this when considering complicate cases. HADOOP-11550
  return prepareEncodingStep(blockGroup);
}

protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
  return blockGroup.getDataBlocks();
}