Java 类com.amazonaws.services.kinesis.model.PutRecordsRequestEntry 实例源码

private void generateRecords()
{
  // Create dummy message
  int recordNo = 1;
  while (recordNo <= sendCount) {
    String dataStr = "Record_" + recordNo;
    PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry();
    putRecordsEntry.setData(ByteBuffer.wrap(dataStr.getBytes()));
    putRecordsEntry.setPartitionKey(dataStr);
    putRecordsRequestEntryList.add(putRecordsEntry);
    if ( (putRecordsRequestEntryList.size() == batchSize) || (recordNo == sendCount )) {
      PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
      putRecordsRequest.setStreamName(streamName);
      putRecordsRequest.setRecords(putRecordsRequestEntryList);
      client.putRecords(putRecordsRequest);
      putRecordsRequestEntryList.clear();
    }
    recordNo++;
  }
}

private void createDummyMessages(String streamName, int count)
        throws Exception
{
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        putRecordsRequestEntry.setData(ByteBuffer.wrap(UUID.randomUUID().toString().getBytes()));
        putRecordsRequestEntry.setPartitionKey(Long.toString(i));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    mockClient.putRecords(putRecordsRequest);
}

private void createJsonMessages(String streamName, int count, int idStart)
        throws Exception
{
    String jsonFormat = "{\"id\" : %d, \"name\" : \"%s\"}";
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        long id = idStart + i;
        String name = UUID.randomUUID().toString();
        String jsonVal = String.format(jsonFormat, id, name);

        // ? with StandardCharsets.UTF_8
        putRecordsRequestEntry.setData(ByteBuffer.wrap(jsonVal.getBytes()));
        putRecordsRequestEntry.setPartitionKey(Long.toString(id));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    mockClient.putRecords(putRecordsRequest);
}

@Override
public PutRecordsResult putRecords(PutRecordsRequest putRecordsRequest) throws AmazonServiceException, AmazonClientException
{
    // Setup method to add a batch of new records:
    InternalStream theStream = this.getStream(putRecordsRequest.getStreamName());
    if (theStream != null) {
        PutRecordsResult result = new PutRecordsResult();
        ArrayList<PutRecordsResultEntry> resultList = new ArrayList<PutRecordsResultEntry>();
        for (PutRecordsRequestEntry entry : putRecordsRequest.getRecords()) {
            PutRecordResult putResult = theStream.putRecord(entry.getData(), entry.getPartitionKey());
            resultList.add((new PutRecordsResultEntry()).withShardId(putResult.getShardId()).withSequenceNumber(putResult.getSequenceNumber()));
        }

        result.setRecords(resultList);
        return result;
    }
    else {
        throw new AmazonClientException("This stream does not exist!");
    }
}

private void createMessages(String streamName, int count)
        throws Exception
{
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        putRecordsRequestEntry.setData(ByteBuffer.wrap(UUID.randomUUID().toString().getBytes()));
        putRecordsRequestEntry.setPartitionKey(Long.toString(i));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    embeddedKinesisStream.getKinesisClient().putRecords(putRecordsRequest);
}

@Test(enabled = false)
public void testKinesisPutRecords() throws Exception {
    List<LogEvent> logEventList = Collections.singletonList(
        Log4jLogEvent.createEvent("foobar", null, "foo.bar", Level.ERROR, null, null, null, null, null, "tname", null, 12345));

    Collection<PutRecordsRequestEntry> records = new ArrayList<>();
    for (LogEvent logEvent : logEventList) {
        String toJson = objectMapper.writeValueAsString(logEvent);
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry()
            .withData(ByteBuffer.wrap(toJson.getBytes()))
            .withPartitionKey("testKinesisPutRecords");
        records.add(putRecordsRequestEntry);
    }

    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setRecords(records);
    putRecordsRequest.setStreamName(awsStreamName);
    PutRecordsResult putRecordsResult = client.putRecords(putRecordsRequest);
    System.out.println(putRecordsResult.toString());
}

/**
 * Converts event to actual kinesis entry type
 */
private PutRecordsRequestEntry toRecordEntries(T event) {
    //Optional<String> partitionKey = partitionKeyTemplate.format(event);
    return new PutRecordsRequestEntry().withData (
            event.raw().asByteBuffer())
            // FIXME: If partitionkey does not return a value, what approach is best?
            .withPartitionKey(partitionKeySupplier.isPresent()
                    ? partitionKeySupplier.get().get().format(event).get()
                    : UUID.randomUUID().toString());
}

/**
 * Based on the request and the result, returns a new request
 * containing the records that failed.
 * @param result is the last PutRecordsResult
 * @return a new PutRecordsRequest with failing records
 */
private PutRecordsRequest failedRecords(PutRecordsResult result) {
    List<PutRecordsRequestEntry> newRecords = new ArrayList<>();
    List<PutRecordsResultEntry> records = result.getRecords();
    for (int i = 0; i < records.size(); i++) {
        if (records.get(i).getErrorCode() != null) {
            newRecords.add(putRecordsRequest.getRecords().get(i));
        }
    }
    return new PutRecordsRequest()
            .withRecords(newRecords)
            .withStreamName(putRecordsRequest.getStreamName());
}

private void addRecord(T tuple)
{
  try {
    Pair<String, V> keyValue = tupleToKeyValue(tuple);
    PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry();
    putRecordsEntry.setData(ByteBuffer.wrap(getRecord(keyValue.second)));
    putRecordsEntry.setPartitionKey(keyValue.first);
    putRecordsRequestEntryList.add(putRecordsEntry);
  } catch (AmazonClientException e) {
    throw new RuntimeException(e);
  }
}

private static Collection<PutRecordsRequestEntry> getRecords(int numRecords) {
  return IntStream.range(0, numRecords).mapToObj(i -> {
    PutRecordsRequestEntry record = new PutRecordsRequestEntry();
    record.setData(ByteBuffer.wrap("{\"a\":1,\"b\":2,\"c\":2}".getBytes(Charsets.UTF_8)));
    record.setPartitionKey(String.valueOf(i));
    return record;
  }).collect(Collectors.toList());
}

public Optional<PutRecordsRequest> put(PutRecordsRequestEntry entry) {
    int newRequestSize = requestSize + entry.getData().remaining()
            + entry.getPartitionKey().length();
    if (entries.size() < maxCount && newRequestSize <= maxSize) {
        requestSize = newRequestSize;
        entries.add(entry);
        return Optional.empty();
    } else {
        Optional<PutRecordsRequest> ret = flush();
        put(entry);
        return ret;
    }
}

@Override
protected void runOnce() throws Exception {
    ClickEvent event = inputQueue.take();
    String partitionKey = event.getSessionId();
    ByteBuffer data = ByteBuffer.wrap(
            event.getPayload().getBytes("UTF-8"));
    recordsPut.getAndIncrement();

    addEntry(new PutRecordsRequestEntry()
            .withPartitionKey(partitionKey)
            .withData(data));
}

protected void addEntry(PutRecordsRequestEntry entry) {
    int newDataSize = dataSize + entry.getData().remaining() +
            entry.getPartitionKey().length();
    if (newDataSize <= 5 * 1024 * 1024 && entries.size() < 500) {
        dataSize = newDataSize;
        entries.add(entry);
    } else {
        flush();
        dataSize = 0;
        addEntry(entry);
    }
}

private static PutRecordsRequestEntry getBadRecord() {
  PutRecordsRequestEntry badRecord = new PutRecordsRequestEntry();
  badRecord.setData(ByteBuffer.wrap("{\"a\":1,\"b\":2,\"c\":2,".getBytes(Charsets.UTF_8)));
  badRecord.setPartitionKey(String.valueOf(0));
  return badRecord;
}