@Test public void testDeleteSecretMaliciousResults() { QueryRequest request = constructQueryRequest(SECRET_NAME); // Result contains entries that do no match the filter in the request (i.e. items for secret2). So it should be // considered malicious. QueryResult maliciousResult = constructQueryResult(true); when(mockDynamoDBClient.query(request)).thenReturn(maliciousResult); // Call the delete secret method. boolean consideredMalicious = false; try { dynamoDB.delete(new SecretIdentifier(SECRET_NAME)); } catch (PotentiallyMaliciousDataException e) { consideredMalicious = true; } assertTrue(consideredMalicious); // Verify nothing was actually deleted because the malicious check failed. verify(mockDynamoDBClient, times(1)).query(request); verify(mockDynamoDBClient, never()).deleteItem(tableName, constructKey(SECRET_NAME, 1)); verify(mockDynamoDBClient, never()).deleteItem(tableName, constructKey(SECRET_NAME, 2)); verify(mockDynamoDBClient, never()).deleteItem(tableName, constructKey(SECRET2_NAME, 1)); }
/** * * @param targetClass <code>Map</code> or entity class with getter/setter method. * @param queryRequest * @return */ public <T> List<T> find(final Class<T> targetClass, final QueryRequest queryRequest) { final QueryResult queryResult = dynamoDB.query(queryRequest); final List<T> res = toList(targetClass, queryResult); if (N.notNullOrEmpty(queryResult.getLastEvaluatedKey()) && N.isNullOrEmpty(queryRequest.getExclusiveStartKey())) { final QueryRequest newQueryRequest = queryRequest.clone(); QueryResult newQueryResult = queryResult; while (N.notNullOrEmpty(newQueryResult.getLastEvaluatedKey())) { newQueryRequest.setExclusiveStartKey(newQueryResult.getLastEvaluatedKey()); newQueryResult = dynamoDB.query(newQueryRequest); res.addAll(toList(targetClass, newQueryResult)); } } return res; }
/** * * @param targetClass * @param queryRequest * @return * @see #find(Class, QueryRequest) */ public DataSet query(final Class<?> targetClass, final QueryRequest queryRequest) { if (targetClass == null || Map.class.isAssignableFrom(targetClass)) { final QueryResult queryResult = dynamoDB.query(queryRequest); final List<Map<String, AttributeValue>> items = queryResult.getItems(); if (N.notNullOrEmpty(queryResult.getLastEvaluatedKey()) && N.isNullOrEmpty(queryRequest.getExclusiveStartKey())) { final QueryRequest newQueryRequest = queryRequest.clone(); QueryResult newQueryResult = queryResult; while (N.notNullOrEmpty(newQueryResult.getLastEvaluatedKey())) { newQueryRequest.setExclusiveStartKey(newQueryResult.getLastEvaluatedKey()); newQueryResult = dynamoDB.query(newQueryRequest); items.addAll(newQueryResult.getItems()); } } return extractData(items, 0, items.size()); } else { return N.newDataSet(find(targetClass, queryRequest)); } }
public RetryResult<QueryResult> queryTable( String tableName, DynamoDBQueryFilter dynamoDBQueryFilter, Map<String, AttributeValue> exclusiveStartKey, long limit, Reporter reporter) { final QueryRequest queryRequest = new QueryRequest() .withTableName(tableName) .withExclusiveStartKey(exclusiveStartKey) .withKeyConditions(dynamoDBQueryFilter.getKeyConditions()) .withLimit(Ints.checkedCast(limit)) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL); RetryResult<QueryResult> retryResult = getRetryDriver().runWithRetry( new Callable<QueryResult>() { @Override public QueryResult call() { log.debug("Executing DynamoDB query: " + queryRequest); return dynamoDB.query(queryRequest); } }, reporter, PrintCounter.DynamoDBReadThrottle); return retryResult; }
public Account accountGetAccountByToken(String token) { Condition hashKeyCondition = new Condition(); hashKeyCondition.withComparisonOperator(ComparisonOperator.EQ).withAttributeValueList(new AttributeValue().withS(token)); Map<String, Condition> keyConditions = new HashMap<String, Condition>(); keyConditions.put("accountToken", hashKeyCondition); QueryRequest queryRequest = new QueryRequest(); queryRequest.withTableName("accountsv2"); queryRequest.withIndexName("accountToken-index"); queryRequest.withKeyConditions(keyConditions); QueryResult result = dynamoDB.query(queryRequest); for(Map<String, AttributeValue> item : result.getItems()) { Account mappedItem = mapper.marshallIntoObject(Account.class, item); // Only want the First one return mappedItem; } return null; }
@Override public void execute() { QueryResult result = ddbClient.query(new QueryRequest() .withTableName(determineTableName()) .withAttributesToGet(determineAttributeNames()) .withConsistentRead(determineConsistentRead()) .withExclusiveStartKey(determineStartKey()) .withKeyConditions(determineKeyConditions()) .withExclusiveStartKey(determineStartKey()) .withLimit(determineLimit()) .withScanIndexForward(determineScanIndexForward())); Map tmp = new HashMap<>(); tmp.put(DdbConstants.ITEMS, result.getItems()); tmp.put(DdbConstants.LAST_EVALUATED_KEY, result.getLastEvaluatedKey()); tmp.put(DdbConstants.CONSUMED_CAPACITY, result.getConsumedCapacity()); tmp.put(DdbConstants.COUNT, result.getCount()); addToResults(tmp); }
public QueryResult query(final QueryRequest request, final int permitsToConsume) throws BackendException { setUserAgent(request); QueryResult result; timedReadThrottle(QUERY, request.getTableName(), permitsToConsume); final Timer.Context apiTimerContext = getTimerContext(QUERY, request.getTableName()); try { result = client.query(request); } catch (Exception e) { throw processDynamoDbApiException(e, QUERY, request.getTableName()); } finally { apiTimerContext.stop(); } meterConsumedCapacity(QUERY, result.getConsumedCapacity()); measureItemCount(QUERY, request.getTableName(), result.getCount()); return result; }
@Override public Map<StaticBuffer, EntryList> getSlice(final List<StaticBuffer> keys, final SliceQuery query, final StoreTransaction txh) throws BackendException { log.debug("Entering getSliceMultiSliceQuery table:{} keys:{} query:{} txh:{}", getTableName(), encodeForLog(keys), encodeForLog(query), txh); final Map<StaticBuffer, EntryList> resultMap = Maps.newHashMapWithExpectedSize(keys.size()); final List<QueryWorker> queryWorkers = Lists.newLinkedList(); for (StaticBuffer hashKey : keys) { final QueryWorker queryWorker = buildQueryWorker(hashKey, query); queryWorkers.add(queryWorker); resultMap.put(hashKey, EntryList.EMPTY_LIST); } final List<QueryResultWrapper> results = client.getDelegate().parallelQuery(queryWorkers); for (QueryResultWrapper resultWrapper : results) { final StaticBuffer titanKey = resultWrapper.getTitanKey(); final QueryResult dynamoDBResult = resultWrapper.getDynamoDBResult(); final EntryList entryList = createEntryListFromItems(dynamoDBResult.getItems(), query); resultMap.put(titanKey, entryList); } log.debug("Exiting getSliceMultiSliceQuery table:{} keys:{} query:{} txh:{} returning:{}", getTableName(), encodeForLog(keys), encodeForLog(query), txh, resultMap.size()); return resultMap; }
@Override public QueryResultWrapper next() throws BackendException { final Query backoff = new ExponentialBackoff.Query(request, delegate, permitsToConsume); final QueryResult result = backoff.runWithBackoff(); final ConsumedCapacity consumedCapacity = result.getConsumedCapacity(); if (null != consumedCapacity) { permitsToConsume = Math.max((int) (consumedCapacity.getCapacityUnits() - 1.0), 1); totalCapacityUnits += consumedCapacity.getCapacityUnits(); } if (result.getLastEvaluatedKey() != null && !result.getLastEvaluatedKey().isEmpty()) { request.setExclusiveStartKey(result.getLastEvaluatedKey()); } else { markComplete(); } // a update returned count returnedCount += result.getCount(); // b update scanned count scannedCount += result.getScannedCount(); // c add scanned finalItemList finalItemList.addAll(result.getItems()); return new QueryResultWrapper(titanKey, result); }
@Override public boolean hasNext() { if (closed) { return false; } if (currentIterator.hasNext()) { return true; } // Loop until the query finds a new result. // This is necessary because even if the query worker has a next page it might have no results. while (queryWorker.hasNext() && !currentIterator.hasNext()) { try { final QueryResultWrapper resultWrapper = queryWorker.next(); final QueryResult queryResult = resultWrapper.getDynamoDBResult(); currentIterator = buildRecordIteratorFromQueryResult(queryResult); } catch (BackendException e) { throw new RuntimeException(e); } } return currentIterator.hasNext(); }
public static List<Map<String, AttributeValue>> queryUntilDone( AmazonDynamoDB dynamoClient, QueryRequest qr, int backoffMillis) throws Exception { List<Map<String, AttributeValue>> output = new ArrayList<>(); Map<String, AttributeValue> lastKeyEvaluated = null; do { int queryAttempts = 0; QueryResult result = null; do { try { result = dynamoClient.query(qr).withLastEvaluatedKey( lastKeyEvaluated); output.addAll(result.getItems()); } catch (ProvisionedThroughputExceededException e) { LOG.warn(String .format("Provisioned Throughput Exceeded - Retry Attempt %s", queryAttempts)); Thread.sleep(2 ^ queryAttempts * backoffMillis); queryAttempts++; } } while (queryAttempts < 10 && result == null); if (result == null) { throw new Exception(String.format( "Unable to execute Query after %s attempts", queryAttempts)); } lastKeyEvaluated = result.getLastEvaluatedKey(); } while (lastKeyEvaluated != null); return output; }
private static void findRepliesForAThread(String forumName, String threadSubject) { String replyId = forumName + "#" + threadSubject; Condition hashKeyCondition = new Condition() .withComparisonOperator(ComparisonOperator.EQ) .withAttributeValueList(new AttributeValue().withS(replyId)); Map<String, Condition> keyConditions = new HashMap<String, Condition>(); keyConditions.put("Id", hashKeyCondition); QueryRequest queryRequest = new QueryRequest() .withTableName(tableName) .withKeyConditions(keyConditions); QueryResult result = client.query(queryRequest); for (Map<String, AttributeValue> item : result.getItems()) { printItem(item); } }
private static void findRepliesForAThreadSpecifyOptionalLimit(String forumName, String threadSubject) { Map<String, AttributeValue> lastEvaluatedKey = null; do { QueryRequest queryRequest = new QueryRequest() .withTableName(tableName) .withKeyConditions(makeReplyKeyConditions(forumName, threadSubject)) .withLimit(1) .withExclusiveStartKey(lastEvaluatedKey); QueryResult result = client.query(queryRequest); for (Map<String, AttributeValue> item : result.getItems()) { printItem(item); } lastEvaluatedKey = result.getLastEvaluatedKey(); } while (lastEvaluatedKey != null); }
private static void findRepliesInLast15DaysWithConfig(String forumName, String threadSubject) { long twoWeeksAgoMilli = (new Date()).getTime() - (15L*24L*60L*60L*1000L); Date twoWeeksAgo = new Date(); twoWeeksAgo.setTime(twoWeeksAgoMilli); SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); String twoWeeksAgoStr = df.format(twoWeeksAgo); Condition rangeKeyCondition = new Condition() .withComparisonOperator(ComparisonOperator.GT.toString()) .withAttributeValueList(new AttributeValue().withS(twoWeeksAgoStr)); Map<String, Condition> keyConditions = makeReplyKeyConditions(forumName, threadSubject); keyConditions.put("ReplyDateTime", rangeKeyCondition); QueryRequest queryRequest = new QueryRequest().withTableName(tableName) .withKeyConditions(keyConditions) .withProjectionExpression("Message, ReplyDateTime, PostedBy"); QueryResult result = client.query(queryRequest); for (Map<String, AttributeValue> item : result.getItems()) { printItem(item); } }
private static void findRepliesUsingAFilterExpression(String forumName, String threadSubject) { Map<String, Condition> keyConditions = makeReplyKeyConditions(forumName, threadSubject); Map<String, AttributeValue> expressionAttributeValues = new HashMap<String, AttributeValue>(); expressionAttributeValues.put(":val", new AttributeValue().withS("User B")); QueryRequest queryRequest = new QueryRequest() .withTableName(tableName) .withKeyConditions(keyConditions) .withFilterExpression("PostedBy = :val") .withExpressionAttributeValues(expressionAttributeValues) .withProjectionExpression("Message, ReplyDateTime, PostedBy"); QueryResult result = client.query(queryRequest); for (Map<String, AttributeValue> item : result.getItems()) { printItem(item); } }
public void run() { QueryRequest queryRequest = DynamoDBUtil.copyQueryRequest(geoQueryRequest.getQueryRequest()); long hashKey = S2Manager.generateHashKey(range.getRangeMin(), config.getHashKeyLength()); List<QueryResult> queryResults = dynamoDBManager.queryGeohash(queryRequest, hashKey, range); for (QueryResult queryResult : queryResults) { if (isInterrupted()) { return; } // getQueryResults() returns a synchronized list. geoQueryResult.getQueryResults().add(queryResult); List<Map<String, AttributeValue>> filteredQueryResult = filter(queryResult.getItems(), geoQueryRequest); // getItem() returns a synchronized list. geoQueryResult.getItem().addAll(filteredQueryResult); } }
public static DynamoEntry getFromMSS(String parameter) { dynamoDB = new AmazonDynamoDBClient(new ProfileCredentialsProvider().getCredentials()); dynamoDB.setRegion(Region.getRegion(Regions.EU_WEST_1)); HashMap<String, Condition> keyConditions = new HashMap<String, Condition>(); keyConditions.put("Parameter", new Condition().withComparisonOperator(ComparisonOperator.EQ) .withAttributeValueList(new AttributeValue(parameter))); QueryRequest queryRequest = new QueryRequest(); queryRequest.setKeyConditions(keyConditions); queryRequest.withTableName("MSS"); QueryResult result = dynamoDB.query(queryRequest); //se houver um pedido ja tratado no MSS getCount > 0, caso contrario nao existe if(result.getCount() == 0) return null; else { //vamos buscar o rank Map<String, AttributeValue> queryResult = result.getItems().get(0); String rank = ""; boolean runningState = false; for (Map.Entry<String, AttributeValue> entry : queryResult.entrySet()) { if(entry.getKey().equals("Rank")) rank = entry.getValue().getN(); else if(entry.getKey().equals("Running")) runningState = entry.getValue().getBOOL(); } DynamoEntry rankResult = new DynamoEntry(new BigInteger(rank), runningState); return rankResult; } }
public static synchronized QueryResult getEntryDynamo(String parameter, AmazonDynamoDBClient dynamoDB) { HashMap<String, Condition> keyConditions = new HashMap<String, Condition>(); keyConditions.put("Parameter", new Condition().withComparisonOperator(ComparisonOperator.EQ) .withAttributeValueList(new AttributeValue(parameter))); QueryRequest queryRequest = new QueryRequest(); queryRequest.setKeyConditions(keyConditions); queryRequest.withTableName("MSS"); QueryResult result = dynamoDB.query(queryRequest); return result; }
@Override public void run() { try { FileInputStream logFile = new FileInputStream("logIntermedio.txt"); BufferedReader bufReader = new BufferedReader(new InputStreamReader(logFile)); String line; while ((line = bufReader.readLine()) != null) { String[] parts = line.split(":"); if (parts[0].equals(threadID)) { AmazonDynamoDBClient dynamoDB = new AmazonDynamoDBClient(new ProfileCredentialsProvider()); dynamoDB.setRegion(Region.getRegion(Regions.EU_WEST_1)); String[] parameterParts = parts[1].split("="); String parameter = parameterParts[1]; QueryResult result = getEntryDynamo(parameter, dynamoDB); if (result.getCount() == 0) // se a query nao retornar nada significa que nao temos nada no dynamo sobre este parametro { putDynamo(parameter, metricas.i_count, metricas.b_count, metricas.fieldLoadCount, metricas.loadCount, dynamoDB, true, threadID, null); } else { //caso contrario vamos buscar as metricas que temos para atualiza las Map<String, AttributeValue> queryResult = result.getItems().get(0); // runningValue fica a true se Running estiver a true, se ja estiver a false entao ja temos as metricas finais para este parametro portanto nao atualizamos o dynamo boolean runningValue = queryResult.values().contains(new AttributeValue().withBOOL(true)); if (runningValue) { putDynamo(parameter, metricas.i_count, metricas.b_count, metricas.fieldLoadCount, metricas.loadCount, dynamoDB, true, threadID, result); } } break; } } } catch (Exception e) { e.printStackTrace(); } }
@Test public void testDeleteSecret() { QueryRequest request = constructQueryRequest(SECRET_NAME); QueryResult result = constructQueryResult(false); when(mockDynamoDBClient.query(request)).thenReturn(result); // Call the delete secret method. dynamoDB.delete(new SecretIdentifier(SECRET_NAME)); // Verify only the entries matching the correct secret were deleted. verify(mockDynamoDBClient, times(1)).query(request); verify(mockDynamoDBClient, times(1)).deleteItem(tableName, constructKey(SECRET_NAME, 1)); verify(mockDynamoDBClient, times(1)).deleteItem(tableName, constructKey(SECRET_NAME, 2)); verify(mockDynamoDBClient, never()).deleteItem(tableName, constructKey(SECRET2_NAME, 1)); }
@Test public void testDeleteSecretNotInTable() { QueryRequest request = constructQueryRequest(SECRET_NAME); QueryResult result = new QueryResult().withCount(0).withItems(new ArrayList<>()); when(mockDynamoDBClient.query(request)).thenReturn(result); // Try deleting the secret. Method should complete without throwing exception but deleteItem will not be // called for any secrets. dynamoDB.delete(new SecretIdentifier(SECRET_NAME)); verify(mockDynamoDBClient, times(1)).query(request); verify(mockDynamoDBClient, never()).deleteItem(tableName, constructKey(SECRET_NAME, 1)); }
@Override protected PageResults<Map<String, AttributeValue>> fetchPage(RequestLimit lim) { // Read from DynamoDB RetryResult<QueryResult> retryResult = context.getClient().queryTable(tableName, context .getSplit().getFilterPushdown(), lastEvaluatedKey, lim.items, context.getReporter()); QueryResult result = retryResult.result; int retries = retryResult.retries; return new PageResults<>(result.getItems(), result.getLastEvaluatedKey(), result .getConsumedCapacity().getCapacityUnits(), retries); }
private QueryOutcome query(Object hk, QuerySpec querySpec, PageIterator pageIterator) { if ( null == _convertMarker ) { throw new IllegalStateException("Index must first be initialized with ConvertMarker"); } if ( pageIterator.getPageSize() <= 0 ) { return new QueryOutcome(new QueryResult()); } ItemCollection<QueryOutcome> itemCollection = maybeBackoff(true, () -> _query.query(withMarker(querySpec.withHashKey(_hkName, hk), pageIterator, hk))); if ( null != itemCollection ) { Iterator<Page<Item, QueryOutcome>> iterator = itemCollection.pages().iterator(); if ( iterator.hasNext() ) { QueryOutcome outcome = maybeBackoff(true, () -> iterator.next().getLowLevelResult()); QueryResult result = outcome.getQueryResult(); if ( null != pageIterator.getMarker() && null != result.getItems() && result.getItems().size() > 0 ) { pageIterator.setPrevMarker(toMarker(result.getItems().get(0), true)); } else { pageIterator.setPrevMarker(null); } Map<String,AttributeValue> lastKey = result.getLastEvaluatedKey(); if ( null != lastKey && ! lastKey.isEmpty() ) { pageIterator.setMarker(toMarker(lastKey, true)); } else { pageIterator.setMarker(null); } return outcome; } } pageIterator.setPrevMarker(null); pageIterator.setMarker(null); return new QueryOutcome(new QueryResult()); }
@SuppressWarnings("unchecked") @Override public QueryResult query(QueryRequest queryRequest) { this.queryRequest = queryRequest; ConsumedCapacity consumed = new ConsumedCapacity(); consumed.setCapacityUnits(1.0); Map<String, AttributeValue> lastEvaluatedKey = new HashMap<String, AttributeValue>(); lastEvaluatedKey.put("1", new AttributeValue("LAST_KEY")); return new QueryResult() .withConsumedCapacity(consumed) .withCount(1) .withItems(getAttributes()) .withLastEvaluatedKey(lastEvaluatedKey); }
@Override protected QueryResultWrapper getMergedPages() { final QueryResult mergedDynamoResult = new QueryResult().withItems(getFinalItemList()) .withCount(returnedCount) .withScannedCount(scannedCount) .withConsumedCapacity(new ConsumedCapacity() .withTableName(request.getTableName()) .withCapacityUnits(totalCapacityUnits)); return new QueryResultWrapper(titanKey, mergedDynamoResult); }
private StaticRecordIterator buildRecordIteratorFromQueryResult(final QueryResult queryResult) { final List<Entry> entries = Lists.newLinkedList(); for (Map<String, AttributeValue> item : queryResult.getItems()) { // DynamoDB's between includes the end of the range, but Titan's slice queries expect the end key to be exclusive final Entry entry = new EntryBuilder(item).slice(rangeKeySliceQuery.getSliceStart(), rangeKeySliceQuery.getSliceEnd()) .build(); if (entry != null) { entries.add(entry); } } return new StaticRecordIterator(entries); }
@Override public List<DColumn> getColumns(String storeName, String rowKey, String startColumn, String endColumn, int count) { Timer t = new Timer(); String key = storeName + "_" + rowKey; HashMap<String,Condition> keyConditions = new HashMap<String,Condition>(); keyConditions.put("key", new Condition() .withComparisonOperator(ComparisonOperator.EQ) .withAttributeValueList(new AttributeValue().withS(key))); if(startColumn != null && endColumn != null) { keyConditions.put("column", new Condition() .withComparisonOperator(ComparisonOperator.BETWEEN) .withAttributeValueList(new AttributeValue().withS(startColumn), new AttributeValue(endColumn))); } else if(startColumn != null) { keyConditions.put("column", new Condition() .withComparisonOperator(ComparisonOperator.GE) .withAttributeValueList(new AttributeValue().withS(startColumn))); } else if(endColumn != null) { keyConditions.put("column", new Condition() .withComparisonOperator(ComparisonOperator.LT) .withAttributeValueList(new AttributeValue().withS(endColumn))); } QueryRequest request = new QueryRequest() .withTableName(getTenant().getName()) .withLimit(Math.min(100, count)) .withKeyConditions(keyConditions); QueryResult result = m_client.query(request); List<DColumn> list = fromItems(result.getItems()); m_logger.debug("get columns range for {} in {}", getTenant().getName(), t); return list; }
public <T> List<T> query(String tableName, String indexName, Map<String, Condition> keyConditions, final ObjectExtractor<T> extractor, String... columnsToInclude) throws EmptyResultDataAccessException { Assert.notNull(tableName, "Table must not be null"); Assert.notNull(extractor, "ObjectExtractor must not be null"); if (logger.isDebugEnabled()) { logger.debug("Executing query on " + tableName + " for " + renderKey(keyConditions)); } QueryRequest request = new QueryRequest(tableName) // .withConsistentRead(false) // because query is used on GSIs [where consistent reads are not supported] - if we needed to query on the primary index could make this a parameter .withKeyConditions(keyConditions); if (columnsToInclude != null && columnsToInclude.length > 0) { request.setAttributesToGet(Arrays.asList(columnsToInclude)); } if (indexName != null) { request.setIndexName(indexName); } QueryResult result = client.query(request); List<Map<String, AttributeValue>> items = result.getItems(); List<T> convertedItems = new ArrayList<T>(items.size()); for (Map<String, AttributeValue> item : items) { convertedItems.add(extractor.extract(item)); } return convertedItems; }
private static void findRepliesPostedWithinTimePeriod(String forumName, String threadSubject) { long startDateMilli = (new Date()).getTime() - (15L*24L*60L*60L*1000L); long endDateMilli = (new Date()).getTime() - (5L*24L*60L*60L*1000L); java.text.SimpleDateFormat df = new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); String startDate = df.format(startDateMilli); String endDate = df.format(endDateMilli); Condition rangeKeyCondition = new Condition() .withComparisonOperator(ComparisonOperator.BETWEEN.toString()) .withAttributeValueList( new AttributeValue().withS(startDate), new AttributeValue().withS(endDate)); Map<String, Condition> keyConditions = makeReplyKeyConditions(forumName, threadSubject); keyConditions.put("ReplyDateTime", rangeKeyCondition); QueryRequest queryRequest = new QueryRequest() .withTableName(tableName) .withKeyConditions(keyConditions) .withProjectionExpression("Message, ReplyDateTime, PostedBy"); QueryResult result = client.query(queryRequest); for (Map<String, AttributeValue> item : result.getItems()) { printItem(item); } }
@Override public <T> PaginatedQueryList<T> query(Class<T> clazz, QueryRequest queryRequest) { QueryResult queryResult = amazonDynamoDB.query(queryRequest); return new PaginatedQueryList<T>(dynamoDBMapper, clazz, amazonDynamoDB, queryRequest, queryResult, dynamoDBMapperConfig.getPaginationLoadingStrategy(), dynamoDBMapperConfig); }
/** * Public ctor. * @param creds Credentials * @param rqst Query request * @param rslt Query result */ NextDosage(final Credentials creds, final QueryRequest rqst, final QueryResult rslt) { this.credentials = creds; this.request = rqst; this.result = rslt; }
@Override public Dosage next() { if (!this.hasNext()) { throw new IllegalStateException( "nothing left in the iterator" ); } final AmazonDynamoDB aws = this.credentials.aws(); try { final QueryRequest rqst = this.request.withExclusiveStartKey( this.result.getLastEvaluatedKey() ); final long start = System.currentTimeMillis(); final QueryResult rslt = aws.query(rqst); Logger.info( this, // @checkstyle LineLength (1 line) "#next(): loaded %d item(s) from '%s' using %s, %s, in %[ms]s", rslt.getCount(), rqst.getTableName(), rqst.getKeyConditions(), new PrintableConsumedCapacity( rslt.getConsumedCapacity() ).print(), System.currentTimeMillis() - start ); return new QueryValve.NextDosage(this.credentials, rqst, rslt); } finally { aws.shutdown(); } }
/** * QueryValve can fetch data from AWS. * @throws Exception If some problem inside */ @Test @SuppressWarnings("unchecked") public void fetchesData() throws Exception { final Valve valve = new QueryValve(); final Credentials credentials = Mockito.mock(Credentials.class); final ImmutableMap<String, AttributeValue> item = new ImmutableMap.Builder<String, AttributeValue>() .build(); final AmazonDynamoDB aws = Mockito.mock(AmazonDynamoDB.class); Mockito.doReturn(aws).when(credentials).aws(); Mockito.doReturn( new QueryResult() .withItems( Collections.<Map<String, AttributeValue>>singletonList(item) ) .withConsumedCapacity( new ConsumedCapacity().withCapacityUnits(1.0d) ) ).when(aws).query(Mockito.any(QueryRequest.class)); final Dosage dosage = valve.fetch( credentials, "table", new Conditions(), new ArrayList<String>(0) ); MatcherAssert.assertThat(dosage.hasNext(), Matchers.is(false)); MatcherAssert.assertThat(dosage.items(), Matchers.hasItem(item)); }
public QueryResult getLatestTweetsForScreenName(String screenName, long timestamp) throws Exception { try { long startDateMilli = System.currentTimeMillis(); Map<String, Condition> keyConditions = new HashMap<String, Condition>(); keyConditions.put( COL_SCREENNAME, new Condition().withComparisonOperator( ComparisonOperator.EQ).withAttributeValueList( new AttributeValue().withS(screenName))); keyConditions.put( COL_CREATEDAT, new Condition().withComparisonOperator( ComparisonOperator.BETWEEN) .withAttributeValueList( new AttributeValue().withN(Long .toString(timestamp)), new AttributeValue().withN(Long .toString(startDateMilli)))); QueryRequest queryRequest = new QueryRequest() .withTableName(TABLE_NAME).withIndexName(INDEX_SCREENNAME) .withKeyConditions(keyConditions) .withSelect(Select.ALL_ATTRIBUTES) .withScanIndexForward(true); QueryResult result = dynamoDB.query(queryRequest); return result; } catch (Exception e) { handleException(e); } return null; }
/** * Query Amazon DynamoDB * * @param hashKey * Hash key for the query request. * * @param range * The range of geohashs to query. * * @return The query result. */ public List<QueryResult> queryGeohash(QueryRequest queryRequest, long hashKey, GeohashRange range) { List<QueryResult> queryResults = new ArrayList<QueryResult>(); Map<String, AttributeValue> lastEvaluatedKey = null; do { Map<String, Condition> keyConditions = new HashMap<String, Condition>(); Condition hashKeyCondition = new Condition().withComparisonOperator(ComparisonOperator.EQ) .withAttributeValueList(new AttributeValue().withN(String.valueOf(hashKey))); keyConditions.put(config.getHashKeyAttributeName(), hashKeyCondition); AttributeValue minRange = new AttributeValue().withN(Long.toString(range.getRangeMin())); AttributeValue maxRange = new AttributeValue().withN(Long.toString(range.getRangeMax())); Condition geohashCondition = new Condition().withComparisonOperator(ComparisonOperator.BETWEEN) .withAttributeValueList(minRange, maxRange); keyConditions.put(config.getGeohashAttributeName(), geohashCondition); queryRequest.withTableName(config.getTableName()).withKeyConditions(keyConditions) .withIndexName(config.getGeohashIndexName()).withConsistentRead(true) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL).withExclusiveStartKey(lastEvaluatedKey); QueryResult queryResult = config.getDynamoDBClient().query(queryRequest); queryResults.add(queryResult); lastEvaluatedKey = queryResult.getLastEvaluatedKey(); } while (lastEvaluatedKey != null); return queryResults; }
public static synchronized void writeLog(String foo) { Metricas aux = metricasThread.get(new Long(Thread.currentThread().getId())); if (!aux.getLast()) { AmazonDynamoDBClient dynamoDB = new AmazonDynamoDBClient(new ProfileCredentialsProvider()); dynamoDB.setRegion(Region.getRegion(Regions.EU_WEST_1)); System.out.println("Thread " + Thread.currentThread().getId() + ": " + aux.i_count + " instructions in " + aux.b_count + " basic blocks were executed " + " field loads: " + aux.fieldLoadCount + " load counts : " + aux.loadCount); aux.setLast(true); contabilizar.put(new Long(Thread.currentThread().getId()), new Integer("1")); try { FileInputStream logFile = new FileInputStream("logIntermedio.txt"); BufferedReader bufReader = new BufferedReader(new InputStreamReader(logFile)); String currentThreadId = "Thread " + Thread.currentThread().getId(); String line; while ((line = bufReader.readLine()) != null) { String[] parts = line.split(":"); String parameter = ""; if (parts[0].equals(currentThreadId)) { String[] parameterParts = parts[1].split("="); parameter = parameterParts[1]; // efetuamos uma query para ir buscar a entrada existente no Dynamo deste parametro // se o estado de Running for true, entao atualizamos a entrada no Dynamo para este parametro // e metemos Running a false. Se o estado retornado pela query for False entao nao fazemos nada // pois isso significa que as metricas para este parametro ja foram calculados. Se for null entao nao // existe entrada e temos que cria-la QueryResult result = getEntryDynamo(parameter, dynamoDB); if (result.getCount() == 0) // se a query nao retora nada significa que nao temos nada no dynamo // sobre este parametro { putDynamo(parameter, aux.i_count, aux.b_count, aux.fieldLoadCount, aux.loadCount, dynamoDB, false, "", null); } else { Map<String, AttributeValue> queryResult = result.getItems().get(0); // runningValue fica a true se Running estiver a true. boolean runningValue = queryResult.values().contains(new AttributeValue().withBOOL(true)); if (runningValue) { putDynamo(parameter, aux.i_count, aux.b_count, aux.fieldLoadCount, aux.loadCount, dynamoDB, false, "", null); } } break; } } } catch (Exception e) {} } else if (contabilizar.get(new Long(Thread.currentThread().getId())).intValue() == 0) { contabilizar.put(new Long(Thread.currentThread().getId()), new Integer("1")); aux.setLast(false); } }
public static synchronized void putDynamo(String parameter, BigInteger instCount, BigInteger basicBlocksCount, BigInteger fieldLoadsCount, BigInteger loadsCount, AmazonDynamoDBClient dynamoDB, boolean runningState, String threadID, QueryResult queryResult) { Map<String, AttributeValue> item = new HashMap<String, AttributeValue>(); item.put("Parameter", new AttributeValue(parameter)); item.put("Instructions", new AttributeValue().withN(instCount.toString())); item.put("Basic Blocks", new AttributeValue().withN(basicBlocksCount.toString())); item.put("Field Loads", new AttributeValue().withN(fieldLoadsCount.toString())); item.put("Loads Stack", new AttributeValue().withN(loadsCount.toString())); item.put("Running", new AttributeValue().withBOOL(runningState)); // calculamos o valor do rank BigDecimal rank = (new BigDecimal(instCount).multiply(new BigDecimal("0.40"))) .add(new BigDecimal(basicBlocksCount).multiply(new BigDecimal("0.15"))) .add(new BigDecimal(fieldLoadsCount).multiply(new BigDecimal("0.05"))) .add(new BigDecimal(loadsCount).multiply(new BigDecimal("0.40"))); item.put("Rank", new AttributeValue().withN(rank.toBigInteger().toString())); // se o runningState vier a true, ou seja e' uma metrica intermedia que vamos atualizar no Dynamo, // temos que tambem introduzir o ThreadID que esta a tratar este parametro e a instancia. // estas duas informacoes sao necessarios para evitar o seguinte caso. Imaginando que duas instancias diferentes // estao a processar o mesmo pedido longo e ambos retiram metricas intermedias. Imaginando que a instancia A comecou // 1 minuto antes, entao nao podemos deixar que a instancia B nao faca overwrite na entrada no Dynamo pois a instancia A tem os // os dados mais recentes. O mesmo se aplica a threads dentro da mesma instancia que processam o mesmo pedido dai precisarmos do ThreadID if (runningState) { // tal como referido acima temos que verificar se esta e' a thread que deve atualizar as metricas intermedias String[] id = threadID.split(" "); if (queryResult != null) { Map<String, AttributeValue> result = queryResult.getItems().get(0); // vamos ver se o id desta thread corresponde ao que deve atualizar as metricas boolean containsID = result.values().contains(new AttributeValue().withN(id[1])); if (containsID) { // se sim temos de ver agora se correspondem 'a mesma instancia pois podem haver threads com o mesmo ID em // maquinas diferentes a processar o mesmo request boolean containsInstance = result.values().contains(new AttributeValue(checkInstanceId())); if (containsInstance) // se sim entao podemos atualizar as metricas { item.put("InstanceID", new AttributeValue(checkInstanceId())); item.put("ThreadID", new AttributeValue().withN((id[1]))); } else //caso contrario nao fazemos nada, o mesmo para o else abaixo return; } else return; } else // este else e' o caso em que nao temos metricas internas deste parametro { item.put("InstanceID", new AttributeValue(checkInstanceId())); item.put("ThreadID", new AttributeValue().withN((id[1]))); } } PutItemRequest putItemRequest = new PutItemRequest().withTableName("MSS").withItem(item); dynamoDB.putItem(putItemRequest); }
private Stream<Entry> query(SecretEventStream.Filter<Entry> filter) { QueryRequest queryRequest = new QueryRequest(); queryRequest.withTableName(tableName); queryRequest.withConsistentRead(true); if (filter.reverse) { queryRequest.setScanIndexForward(false); } KeyExpressionGenerator keyExpressionGenerator = new KeyExpressionGenerator(); KeyExpressionGenerator.KeyCondition keyExpression = keyExpressionGenerator.process(filter.parsedKeyCondition.get(), converters); Map<String, String> expressionAttributeNames = keyExpression.expressionAttributeNames; Map<String, AttributeValue> expressionAttributeValues = keyExpression.expressionAttributeValues; if (filter.parsedAttributeCondition.isPresent()) { FilterGenerator filterGenerator = new FilterGenerator(expressionAttributeValues.size()+1); FilterGenerator.Filter generated = filterGenerator.process(filter.parsedAttributeCondition.get(), converters); if(!generated.expressionAttributeNames.isEmpty()) { expressionAttributeNames = FilterGenerator.merge(expressionAttributeNames, generated.expressionAttributeNames); } if (!generated.expressionAttributeValues.isEmpty()) { expressionAttributeValues = FilterGenerator.merge(expressionAttributeValues, generated.expressionAttributeValues); } queryRequest.withFilterExpression(generated.filterExpression); } queryRequest.withExpressionAttributeNames(expressionAttributeNames); queryRequest.withExpressionAttributeValues(expressionAttributeValues); queryRequest.withKeyConditionExpression(keyExpression.keyConditionExpression); QueryResult result = client.query(queryRequest); List<Map<String, AttributeValue>> results = new ArrayList<>(); results.addAll(result.getItems()); while (result.getLastEvaluatedKey() != null) { queryRequest = queryRequest.withExclusiveStartKey(result.getLastEvaluatedKey()); result = client.query(queryRequest); results.addAll(result.getItems()); } return results.stream().map(this::fromMap); }
private static QueryResult constructQueryResult(boolean includeSecret2) { ArrayList<Map<String, AttributeValue>> items = constructItems(includeSecret2); return new QueryResult().withItems(items).withCount(items.size()).withScannedCount(items.size()); }