Java 类org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder 实例源码

项目：hops 文件：HHXORErasureDecoder.java

@Override
protected ErasureCodingStep prepareDecodingStep(
        final ECBlockGroup blockGroup) {

  RawErasureDecoder rawDecoder;
  RawErasureEncoder rawEncoder;

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);
  ECBlock[] outputBlocks = getOutputBlocks(blockGroup);

  rawDecoder = checkCreateRSRawDecoder();
  rawEncoder = checkCreateXorRawEncoder();

  return new HHXORErasureDecodingStep(inputBlocks,
          getErasedIndexes(inputBlocks), outputBlocks, rawDecoder,
          rawEncoder);
}

项目：hops 文件：TestCodecRawCoderMapping.java

@Test
public void testRSDefaultRawCoder() {
  ErasureCoderOptions coderOptions = new ErasureCoderOptions(
      numDataUnit, numParityUnit);
  // should return default raw coder of rs-default codec
  RawErasureEncoder encoder = CodecUtil.createRawEncoder(
      conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
  Assert.assertTrue(encoder instanceof RSRawEncoder);
  RawErasureDecoder decoder = CodecUtil.createRawDecoder(
      conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
  Assert.assertTrue(decoder instanceof RSRawDecoder);

  // should return default raw coder of rs-legacy codec
  encoder = CodecUtil.createRawEncoder(conf,
      ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
  Assert.assertTrue(encoder instanceof RSRawEncoderLegacy);
  decoder = CodecUtil.createRawDecoder(conf,
      ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
  Assert.assertTrue(decoder instanceof RSRawDecoderLegacy);
}

项目：hops 文件：TestCodecRawCoderMapping.java

@Test
public void testDedicatedRawCoderKey() {
  ErasureCoderOptions coderOptions = new ErasureCoderOptions(
      numDataUnit, numParityUnit);

  String dummyFactName = "DummyNoneExistingFactory";
  // set the dummy factory to rs-legacy and create a raw coder
  // with rs-default, which is OK as the raw coder key is not used
  conf.set(CommonConfigurationKeys.
      IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName);
  RawErasureEncoder encoder = CodecUtil.createRawEncoder(conf,
      ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
  Assert.assertTrue(encoder instanceof RSRawEncoder);
  // now create the raw coder with rs-legacy, which should throw exception
  try {
    CodecUtil.createRawEncoder(conf,
        ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
    Assert.fail();
  } catch (Exception e) {
    GenericTestUtils.assertExceptionContains("Failed to create raw coder", e);
  }
}

项目：hadoop-oss 文件：HHXORErasureDecodingStep.java

/**
 * The constructor with all the necessary info.
 * @param inputBlocks
 * @param erasedIndexes the indexes of erased blocks in inputBlocks array
 * @param outputBlocks
 * @param rawDecoder underlying RS decoder for hitchhiker decoding
 * @param rawEncoder underlying XOR encoder for hitchhiker decoding
 */
public HHXORErasureDecodingStep(ECBlock[] inputBlocks, int[] erasedIndexes,
    ECBlock[] outputBlocks, RawErasureDecoder rawDecoder,
    RawErasureEncoder rawEncoder) {
  super(inputBlocks, outputBlocks);
  this.pbIndex = rawDecoder.getNumParityUnits() - 1;
  this.erasedIndexes = erasedIndexes;
  this.rsRawDecoder = rawDecoder;
  this.xorRawEncoder = rawEncoder;

  this.piggyBackIndex = HHUtil.initPiggyBackIndexWithoutPBVec(
      rawDecoder.getNumDataUnits(), rawDecoder.getNumParityUnits());
  this.piggyBackFullIndex = HHUtil.initPiggyBackFullIndexVec(
      rawDecoder.getNumDataUnits(), piggyBackIndex);
}

项目：hadoop-oss 文件：HHXORErasureEncoder.java

@Override
protected ErasureCodingStep prepareEncodingStep(
        final ECBlockGroup blockGroup) {

  RawErasureEncoder rsRawEncoderTmp = checkCreateRSRawEncoder();
  RawErasureEncoder xorRawEncoderTmp = checkCreateXorRawEncoder();

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new HHXORErasureEncodingStep(inputBlocks,
          getOutputBlocks(blockGroup), rsRawEncoderTmp, xorRawEncoderTmp);
}

项目：hadoop-oss 文件：HHXORErasureEncoder.java

private RawErasureEncoder checkCreateRSRawEncoder() {
  if (rsRawEncoder == null) {
    rsRawEncoder = CodecUtil.createRSRawEncoder(getConf(),
            getNumDataUnits(), getNumParityUnits());
  }
  return rsRawEncoder;
}

项目：hadoop-oss 文件：HHXORErasureEncoder.java

private RawErasureEncoder checkCreateXorRawEncoder() {
  if (xorRawEncoder == null) {
    xorRawEncoder = CodecUtil.createXORRawEncoder(getConf(),
            getNumDataUnits(), getNumParityUnits());
    xorRawEncoder.setCoderOption(CoderOption.ALLOW_CHANGE_INPUTS, false);
  }
  return xorRawEncoder;
}

项目：hadoop-oss 文件：RSErasureEncoder.java

private RawErasureEncoder checkCreateRSRawEncoder() {
  if (rawEncoder == null) {
    rawEncoder = CodecUtil.createRSRawEncoder(getConf(),
            getNumDataUnits(), getNumParityUnits());
  }
  return rawEncoder;
}

项目：hadoop-oss 文件：XORErasureEncoder.java

@Override
protected ErasureCodingStep prepareEncodingStep(
    final ECBlockGroup blockGroup) {
  RawErasureEncoder rawEncoder = CodecUtil.createXORRawEncoder(getConf(),
      getNumDataUnits(), getNumParityUnits());

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new ErasureEncodingStep(inputBlocks,
      getOutputBlocks(blockGroup), rawEncoder);
}

项目：hadoop-oss 文件：HHXORErasureEncodingStep.java

/**
 * The constructor with all the necessary info.
 *
 * @param inputBlocks
 * @param outputBlocks
 * @param rsRawEncoder  underlying RS encoder for hitchhiker encoding
 * @param xorRawEncoder underlying XOR encoder for hitchhiker encoding
 */
public HHXORErasureEncodingStep(ECBlock[] inputBlocks, ECBlock[] outputBlocks,
                                RawErasureEncoder rsRawEncoder,
                                RawErasureEncoder xorRawEncoder) {
  super(inputBlocks, outputBlocks);

  this.rsRawEncoder = rsRawEncoder;
  this.xorRawEncoder = xorRawEncoder;
  piggyBackIndex = HHUtil.initPiggyBackIndexWithoutPBVec(
          rsRawEncoder.getNumDataUnits(), rsRawEncoder.getNumParityUnits());
}

项目：hadoop-oss 文件：CodecUtil.java

/**
 * Create RS raw encoder according to configuration.
 * @param conf configuration possibly with some items to configure the coder
 * @param numDataUnits number of data units in a coding group
 * @param numParityUnits number of parity units in a coding group
 * @return raw encoder
 */
public static RawErasureEncoder createRSRawEncoder(
    Configuration conf, int numDataUnits, int numParityUnits) {
  RawErasureCoder rawCoder = createRawCoder(conf,
      CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
      true, numDataUnits, numParityUnits);
  if (rawCoder == null) {
    rawCoder = new RSRawEncoder(numDataUnits, numParityUnits);
  }

  return (RawErasureEncoder) rawCoder;
}

项目：hadoop-oss 文件：CodecUtil.java

/**
 * Create XOR raw encoder according to configuration.
 * @param conf configuration possibly with some items to configure the coder
 * @param numDataUnits number of data units in a coding group
 * @param numParityUnits number of parity units in a coding group
 * @return raw encoder
 */
public static RawErasureEncoder createXORRawEncoder(
    Configuration conf, int numDataUnits, int numParityUnits) {
  RawErasureCoder rawCoder = createRawCoder(conf,
      CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
      true, numDataUnits, numParityUnits);
  if (rawCoder == null) {
    rawCoder = new XORRawEncoder(numDataUnits, numParityUnits);
  }

  return (RawErasureEncoder) rawCoder;
}

项目：aliyun-oss-hadoop-fs 文件：DFSStripedOutputStream.java

/**
 * Encode the buffers, i.e. compute parities.
 *
 * @param buffers data buffers + parity buffers
 */
private static void encode(RawErasureEncoder encoder, int numData,
    ByteBuffer[] buffers) {
  final ByteBuffer[] dataBuffers = new ByteBuffer[numData];
  final ByteBuffer[] parityBuffers = new ByteBuffer[buffers.length - numData];
  System.arraycopy(buffers, 0, dataBuffers, 0, dataBuffers.length);
  System.arraycopy(buffers, numData, parityBuffers, 0, parityBuffers.length);

  encoder.encode(dataBuffers, parityBuffers);
}

项目：aliyun-oss-hadoop-fs 文件：StripedFileTestUtil.java

static void verifyParityBlocks(Configuration conf, final long size,
    final int cellSize, byte[][] dataBytes, byte[][] parityBytes,
    Set<Integer> checkSet) {
  // verify the parity blocks
  int parityBlkSize = (int) StripedBlockUtil.getInternalBlockLength(
      size, cellSize, dataBytes.length, dataBytes.length);
  final byte[][] expectedParityBytes = new byte[parityBytes.length][];
  for (int i = 0; i < parityBytes.length; i++) {
    expectedParityBytes[i] = new byte[parityBlkSize];
  }
  for (int i = 0; i < dataBytes.length; i++) {
    if (dataBytes[i] == null) {
      dataBytes[i] = new byte[dataBytes[0].length];
    } else if (dataBytes[i].length < dataBytes[0].length) {
      final byte[] tmp = dataBytes[i];
      dataBytes[i] = new byte[dataBytes[0].length];
      System.arraycopy(tmp, 0, dataBytes[i], 0, tmp.length);
    }
  }
  final RawErasureEncoder encoder =
      CodecUtil.createRSRawEncoder(conf, dataBytes.length, parityBytes.length);
  encoder.encode(dataBytes, expectedParityBytes);
  for (int i = 0; i < parityBytes.length; i++) {
    if (checkSet.contains(i + dataBytes.length)){
      Assert.assertArrayEquals("i=" + i, expectedParityBytes[i],
          parityBytes[i]);
    }
  }
}

项目：aliyun-oss-hadoop-fs 文件：RSErasureEncoder.java

private RawErasureEncoder checkCreateRSRawEncoder() {
  if (rawEncoder == null) {
    rawEncoder = CodecUtil.createRSRawEncoder(getConf(),
            getNumDataUnits(), getNumParityUnits());
  }
  return rawEncoder;
}

项目：aliyun-oss-hadoop-fs 文件：XORErasureEncoder.java

@Override
protected ErasureCodingStep prepareEncodingStep(
    final ECBlockGroup blockGroup) {
  RawErasureEncoder rawEncoder = CodecUtil.createXORRawEncoder(getConf(),
      getNumDataUnits(), getNumParityUnits());

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new ErasureEncodingStep(inputBlocks,
      getOutputBlocks(blockGroup), rawEncoder);
}

项目：hops 文件：HHXORErasureDecodingStep.java

/**
 * The constructor with all the necessary info.
 * @param inputBlocks
 * @param erasedIndexes the indexes of erased blocks in inputBlocks array
 * @param outputBlocks
 * @param rawDecoder underlying RS decoder for hitchhiker decoding
 * @param rawEncoder underlying XOR encoder for hitchhiker decoding
 */
public HHXORErasureDecodingStep(ECBlock[] inputBlocks, int[] erasedIndexes,
    ECBlock[] outputBlocks, RawErasureDecoder rawDecoder,
    RawErasureEncoder rawEncoder) {
  super(inputBlocks, outputBlocks);
  this.pbIndex = rawDecoder.getNumParityUnits() - 1;
  this.erasedIndexes = erasedIndexes;
  this.rsRawDecoder = rawDecoder;
  this.xorRawEncoder = rawEncoder;

  this.piggyBackIndex = HHUtil.initPiggyBackIndexWithoutPBVec(
      rawDecoder.getNumDataUnits(), rawDecoder.getNumParityUnits());
  this.piggyBackFullIndex = HHUtil.initPiggyBackFullIndexVec(
      rawDecoder.getNumDataUnits(), piggyBackIndex);
}

项目：hops 文件：HHXORErasureDecoder.java

private RawErasureEncoder checkCreateXorRawEncoder() {
  if (xorRawEncoder == null) {
    ErasureCoderOptions coderOptions = new ErasureCoderOptions(
        getNumDataUnits(), getNumParityUnits());
    xorRawEncoder = CodecUtil.createRawEncoder(getConf(),
        ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
  }
  return xorRawEncoder;
}

项目：hops 文件：HHXORErasureEncoder.java

@Override
protected ErasureCodingStep prepareEncodingStep(
        final ECBlockGroup blockGroup) {

  RawErasureEncoder rsRawEncoderTmp = checkCreateRSRawEncoder();
  RawErasureEncoder xorRawEncoderTmp = checkCreateXorRawEncoder();

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new HHXORErasureEncodingStep(inputBlocks,
          getOutputBlocks(blockGroup), rsRawEncoderTmp, xorRawEncoderTmp);
}

项目：hops 文件：HHXORErasureEncoder.java

private RawErasureEncoder checkCreateRSRawEncoder() {
  if (rsRawEncoder == null) {
    ErasureCoderOptions coderOptions = new ErasureCoderOptions(
        getNumDataUnits(), getNumParityUnits());
    rsRawEncoder = CodecUtil.createRawEncoder(getConf(),
        ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
  }
  return rsRawEncoder;
}

项目：hops 文件：HHXORErasureEncoder.java

private RawErasureEncoder checkCreateXorRawEncoder() {
  if (xorRawEncoder == null) {
    ErasureCoderOptions erasureCoderOptions = new ErasureCoderOptions(
        getNumDataUnits(), getNumParityUnits());
    xorRawEncoder = CodecUtil.createRawEncoder(getConf(),
        ErasureCodeConstants.XOR_CODEC_NAME,
        erasureCoderOptions);
  }
  return xorRawEncoder;
}

项目：hops 文件：RSErasureEncoder.java

private RawErasureEncoder checkCreateRSRawEncoder() {
  if (rawEncoder == null) {
    // TODO: we should create the raw coder according to codec.
    ErasureCoderOptions coderOptions = new ErasureCoderOptions(
        getNumDataUnits(), getNumParityUnits());
    rawEncoder = CodecUtil.createRawEncoder(getConf(),
        ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
  }
  return rawEncoder;
}

项目：hops 文件：XORErasureEncoder.java

@Override
protected ErasureCodingStep prepareEncodingStep(
    final ECBlockGroup blockGroup) {
  ErasureCoderOptions coderOptions = new ErasureCoderOptions(
      getNumDataUnits(), getNumParityUnits());
  RawErasureEncoder rawEncoder = CodecUtil.createRawEncoder(getConf(),
      ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new ErasureEncodingStep(inputBlocks,
      getOutputBlocks(blockGroup), rawEncoder);
}

项目：hops 文件：HHXORErasureEncodingStep.java

/**
 * The constructor with all the necessary info.
 *
 * @param inputBlocks
 * @param outputBlocks
 * @param rsRawEncoder  underlying RS encoder for hitchhiker encoding
 * @param xorRawEncoder underlying XOR encoder for hitchhiker encoding
 */
public HHXORErasureEncodingStep(ECBlock[] inputBlocks, ECBlock[] outputBlocks,
                                RawErasureEncoder rsRawEncoder,
                                RawErasureEncoder xorRawEncoder) {
  super(inputBlocks, outputBlocks);

  this.rsRawEncoder = rsRawEncoder;
  this.xorRawEncoder = xorRawEncoder;
  piggyBackIndex = HHUtil.initPiggyBackIndexWithoutPBVec(
          rsRawEncoder.getNumDataUnits(), rsRawEncoder.getNumParityUnits());
}

项目：hops 文件：CodecUtil.java

/**
 * Create RS raw encoder according to configuration.
 * @param conf configuration
 * @param coderOptions coder options that's used to create the coder
 * @param codec the codec to use. If null, will use the default codec
 * @return raw encoder
 */
public static RawErasureEncoder createRawEncoder(
    Configuration conf, String codec, ErasureCoderOptions coderOptions) {
  Preconditions.checkNotNull(conf);
  Preconditions.checkNotNull(codec);

  String rawCoderFactoryKey = getFactNameFromCodec(conf, codec);

  RawErasureCoderFactory fact = createRawCoderFactory(conf,
      rawCoderFactoryKey);

  return fact.createEncoder(coderOptions);
}

项目：hadoop-oss 文件：HHUtil.java

public static ByteBuffer[] getPiggyBacksFromInput(ByteBuffer[] inputs,
                                                  int[] piggyBackIndex,
                                                  int numParityUnits,
                                                  int pgIndex,
                                                  RawErasureEncoder encoder) {
  ByteBuffer[] emptyInput = new ByteBuffer[inputs.length];
  ByteBuffer[] tempInput = new ByteBuffer[inputs.length];
  int[] inputPositions = new int[inputs.length];

  for (int m = 0; m < inputs.length; ++m) {
    if (inputs[m] != null) {
      emptyInput[m] = allocateByteBuffer(inputs[m].isDirect(),
              inputs[m].remaining());
    }
  }

  ByteBuffer[] tempOutput = new ByteBuffer[numParityUnits];
  for (int m = 0; m < numParityUnits; ++m) {
    tempOutput[m] = allocateByteBuffer(inputs[m].isDirect(),
            inputs[0].remaining());
  }

  ByteBuffer[] piggyBacks = new ByteBuffer[numParityUnits - 1];
  assert (piggyBackIndex.length >= numParityUnits);

  // using underlying RS code to create piggybacks
  for (int i = 0; i < numParityUnits - 1; ++i) {
    for (int k = piggyBackIndex[i]; k < piggyBackIndex[i + 1]; ++k) {
      tempInput[k] = inputs[k];
      inputPositions[k] = inputs[k].position();
    }
    for (int n = 0; n < emptyInput.length; ++n) {
      if (tempInput[n] == null) {
        tempInput[n] = emptyInput[n];
        inputPositions[n] = emptyInput[n].position();
      }
    }

    encoder.encode(tempInput, tempOutput);

    piggyBacks[i] = cloneBufferData(tempOutput[pgIndex]);

    for (int j = 0; j < tempInput.length; j++) {
      if (tempInput[j] != null) {
        tempInput[j].position(inputPositions[j]);
        tempInput[j] = null;
      }
    }

    for (int j = 0; j < tempOutput.length; j++) {
      tempOutput[j].clear();
    }
  }

  return piggyBacks;
}

项目：hops 文件：HHUtil.java

public static ByteBuffer[] getPiggyBacksFromInput(ByteBuffer[] inputs,
                                                  int[] piggyBackIndex,
                                                  int numParityUnits,
                                                  int pgIndex,
                                                  RawErasureEncoder encoder) {
  ByteBuffer[] emptyInput = new ByteBuffer[inputs.length];
  ByteBuffer[] tempInput = new ByteBuffer[inputs.length];
  int[] inputPositions = new int[inputs.length];

  for (int m = 0; m < inputs.length; ++m) {
    if (inputs[m] != null) {
      emptyInput[m] = allocateByteBuffer(inputs[m].isDirect(),
              inputs[m].remaining());
    }
  }

  ByteBuffer[] tempOutput = new ByteBuffer[numParityUnits];
  for (int m = 0; m < numParityUnits; ++m) {
    tempOutput[m] = allocateByteBuffer(inputs[m].isDirect(),
            inputs[0].remaining());
  }

  ByteBuffer[] piggyBacks = new ByteBuffer[numParityUnits - 1];
  assert (piggyBackIndex.length >= numParityUnits);

  // using underlying RS code to create piggybacks
  for (int i = 0; i < numParityUnits - 1; ++i) {
    for (int k = piggyBackIndex[i]; k < piggyBackIndex[i + 1]; ++k) {
      tempInput[k] = inputs[k];
      inputPositions[k] = inputs[k].position();
    }
    for (int n = 0; n < emptyInput.length; ++n) {
      if (tempInput[n] == null) {
        tempInput[n] = emptyInput[n];
        inputPositions[n] = emptyInput[n].position();
      }
    }

    encoder.encode(tempInput, tempOutput);

    piggyBacks[i] = cloneBufferData(tempOutput[pgIndex]);

    for (int j = 0; j < tempInput.length; j++) {
      if (tempInput[j] != null) {
        tempInput[j].position(inputPositions[j]);
        tempInput[j] = null;
      }
    }

    for (int j = 0; j < tempOutput.length; j++) {
      tempOutput[j].clear();
    }
  }

  return piggyBacks;
}

项目：hadoop-oss 文件：RSErasureEncoder.java

@Override
protected ErasureCodingStep prepareEncodingStep(final ECBlockGroup blockGroup) {

  RawErasureEncoder rawEncoder = checkCreateRSRawEncoder();

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new ErasureEncodingStep(inputBlocks,
      getOutputBlocks(blockGroup), rawEncoder);
}

项目：aliyun-oss-hadoop-fs 文件：RSErasureEncoder.java

@Override
protected ErasureCodingStep prepareEncodingStep(final ECBlockGroup blockGroup) {

  RawErasureEncoder rawEncoder = checkCreateRSRawEncoder();

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new ErasureEncodingStep(inputBlocks,
      getOutputBlocks(blockGroup), rawEncoder);
}

项目：hops 文件：RSErasureEncoder.java

@Override
protected ErasureCodingStep prepareEncodingStep(final ECBlockGroup blockGroup) {

  RawErasureEncoder rawEncoder = checkCreateRSRawEncoder();

  ECBlock[] inputBlocks = getInputBlocks(blockGroup);

  return new ErasureEncodingStep(inputBlocks,
      getOutputBlocks(blockGroup), rawEncoder);
}

项目：hadoop-oss 文件：ErasureEncodingStep.java

/**
 * The constructor with all the necessary info.
 * @param inputBlocks
 * @param outputBlocks
 * @param rawEncoder
 */
public ErasureEncodingStep(ECBlock[] inputBlocks, ECBlock[] outputBlocks,
                           RawErasureEncoder rawEncoder) {
  super(inputBlocks, outputBlocks);
  this.rawEncoder = rawEncoder;
}

项目：aliyun-oss-hadoop-fs 文件：ErasureEncodingStep.java

/**
 * The constructor with all the necessary info.
 * @param inputBlocks
 * @param outputBlocks
 * @param rawEncoder
 */
public ErasureEncodingStep(ECBlock[] inputBlocks, ECBlock[] outputBlocks,
                           RawErasureEncoder rawEncoder) {
  super(inputBlocks, outputBlocks);
  this.rawEncoder = rawEncoder;
}

项目：hops 文件：ErasureEncodingStep.java

/**
 * The constructor with all the necessary info.
 * @param inputBlocks
 * @param outputBlocks
 * @param rawEncoder
 */
public ErasureEncodingStep(ECBlock[] inputBlocks, ECBlock[] outputBlocks,
                           RawErasureEncoder rawEncoder) {
  super(inputBlocks, outputBlocks);
  this.rawEncoder = rawEncoder;
}