@Override public Iterable<Directive> details(final long time) throws IOException { final Iterator<Item> items = this.table() .frame() .through( new QueryValve() .withSelect(Select.ALL_ATTRIBUTES) .withLimit(1) ) .where("url", Conditions.equalTo(this.url)) .where("time", Conditions.equalTo(time)) .iterator(); if (!items.hasNext()) { throw new IllegalArgumentException( String.format( "Request at %d for %s not found", time, this.url ) ); } return DyStatus.xembly(items.next(), true); }
@Override public Iterable<Take> expired() { return new Mapped<>( this.table() .frame() .through( new QueryValve() .withIndexName("expired") .withConsistentRead(false) .withSelect(Select.ALL_ATTRIBUTES) ) .where("success", Conditions.equalTo(Boolean.toString(false))) .where( "when", new Condition() .withComparisonOperator(ComparisonOperator.LT) .withAttributeValueList( new AttributeValue().withN( Long.toString(System.currentTimeMillis()) ) ) ), item -> new DyTake(item, this.delay) ); }
@Override public Iterable<Iterable<Directive>> scripts() { return new Mapped<Item, Iterable<Directive>>( item -> new Directives() .add("script") .add("name").set(item.get("name").getS()).up() .add("bash").set(item.get("bash").getS()).up() .add("paid").set(item.get("paid").getN()).up() .add("used").set(item.get("used").getN()).up() .add("hour").set(item.get("hour").getN()).up() .add("day").set(item.get("day").getN()).up() .add("week").set(item.get("week").getN()).up() .up(), this.region.table("scripts") .frame() .through( new QueryValve() // @checkstyle MagicNumber (1 line) .withLimit(10) .withSelect(Select.ALL_ATTRIBUTES) ) .where("login", this.login) ); }
@Override public Iterable<Iterable<Directive>> logs() { return new Mapped<Item, Iterable<Directive>>( item -> new Directives() .add("log") .add("group").set(item.get("group").getS()).up() .add("start").set(item.get("start").getN()).up() .add("finish").set(item.get("finish").getN()).up() .add("period").set(item.get("period").getS()).up() .add("ocket").set(item.get("ocket").getS()).up() .add("exit").set(item.get("exit").getN()).up() .up(), this.region.table("logs") .frame() .through( new QueryValve() .withIndexName("mine") // @checkstyle MagicNumber (1 line) .withLimit(20) .withConsistentRead(false) .withScanIndexForward(false) .withSelect(Select.ALL_ATTRIBUTES) ) .where("login", this.login) ); }
@Override public Iterable<Domain> mine() { return this.table() .frame() .through( new QueryValve() .withSelect(Select.ALL_ATTRIBUTES) .withLimit(Tv.HUNDRED) .withConsistentRead(false) .withIndexName("mine") ) .where("user", Conditions.equalTo(this.handle)) .stream() .map(DyDomain::new) .map(Domain.class::cast) .collect(Collectors.toList()); }
@Override public Iterable<Domain> domain(final String name) { return this.table() .frame() .through( new QueryValve() .withSelect(Select.ALL_ATTRIBUTES) .withLimit(1) .withConsistentRead(true) ) .where( "domain", Conditions.equalTo(name.toLowerCase(Locale.ENGLISH)) ) .stream() .map(DyDomain::new) .map(Domain.class::cast) .collect(Collectors.toList()); }
@Override public Iterable<Pipe> iterate() { return () -> this.table() .frame() .through( new QueryValve() .withSelect(Select.ALL_ATTRIBUTES) .withLimit(Tv.HUNDRED) .withConsistentRead(true) ) .where("urn", Conditions.equalTo(this.urn)) .stream() .map(DyPipe::new) .map(Pipe.class::cast) .iterator(); }
@Override public Iterable<Event> iterate() { return () -> this.table() .frame() .through( new QueryValve() .withLimit(Tv.TWENTY) .withIndexName("top") .withSelect(Select.ALL_ATTRIBUTES) .withScanIndexForward(false) .withConsistentRead(false) ) .where("urn", Conditions.equalTo(this.urn)) .stream() .map(DyEvent::new) .map(Event.class::cast) .iterator(); }
/** * Generate a list of attribute names found in the Aggregator's dynamo * table. Assumes that all Items in the Aggregator table are of the same * structure. * * @param dynamoClient * Dynamo DB Client to use for connection to Dynamo DB. * @param dynamoTable * The Dynamo Table for the Aggregator * @return A list of attribute names from the Dynamo table * @throws Exception */ public static List<String> getDictionaryEntry( final AmazonDynamoDB dynamoClient, final String dynamoTable) throws Exception { // get a list of all columns in the table, with keys first List<String> columns = new ArrayList<>(); List<KeySchemaElement> keys = dynamoClient.describeTable(dynamoTable) .getTable().getKeySchema(); for (KeySchemaElement key : keys) { columns.add(key.getAttributeName()); } ScanResult scan = dynamoClient.scan(new ScanRequest() .withTableName(dynamoTable).withSelect(Select.ALL_ATTRIBUTES) .withLimit(1)); List<Map<String, AttributeValue>> scannedItems = scan.getItems(); for (Map<String, AttributeValue> map : scannedItems) { for (String s : map.keySet()) { if (!columns.contains(s)) columns.add(s); } } return columns; }
/** * Generate a list of attribute names found in the Aggregator's dynamo * table. Assumes that all Items in the Aggregator table are of the same * structure. * * @param dynamoClient Dynamo DB Client to use for connection to Dynamo DB. * @param dynamoTable The Dynamo Table for the Aggregator * @return A list of attribute names from the Dynamo table * @throws Exception */ protected List<String> getDictionaryEntry() throws Exception { // get a list of all columns in the table, with keys first List<String> columns = new ArrayList<>(); List<KeySchemaElement> keys = dynamoClient.describeTable(this.tableName).getTable().getKeySchema(); for (KeySchemaElement key : keys) { columns.add(key.getAttributeName()); } ScanResult scan = dynamoClient.scan(new ScanRequest().withTableName(this.tableName).withSelect( Select.ALL_ATTRIBUTES).withLimit(1)); List<Map<String, AttributeValue>> scannedItems = scan.getItems(); for (Map<String, AttributeValue> map : scannedItems) { for (String s : map.keySet()) { if (!columns.contains(s)) columns.add(s); } } return columns; }
@Override public Iterable<Deck> iterate() { return Iterables.transform( this.region.table(DyDeck.TBL) .frame() .through(new QueryValve().withSelect(Select.ALL_ATTRIBUTES)) .where(DyDeck.HASH, this.user), new Function<Item, Deck>() { @Override public Deck apply(final Item input) { try { return DyDecks.this.get(input.get(DyDeck.RANGE).getS()); } catch (final IOException ex) { throw new IllegalStateException(ex); } } } ); }
/** * Item. * @return Item */ private Item item() { final Iterator<Item> items = this.region .table(DyDeck.TBL) .frame() .through( new QueryValve() .withSelect(Select.ALL_ATTRIBUTES) .withLimit(1) ) .where(DyDeck.HASH, this.user) .where(DyDeck.RANGE, this.deck) .iterator(); if (!items.hasNext()) { throw new IllegalArgumentException( String.format("deck '%s' not found", this.deck) ); } return items.next(); }
@Override public Take target(final URL url, final long time) throws IOException { final Iterator<Item> items = this.table() .frame() .through( new QueryValve() .withSelect(Select.ALL_ATTRIBUTES) .withLimit(1) ) .where("url", Conditions.equalTo(url)) .where("time", Conditions.equalTo(time)) .iterator(); final Item item; if (items.hasNext()) { item = items.next(); } else { item = this.table().put( new Attributes() .with("url", url) .with("time", time) .with("code", 0) .with("attempts", 0) .with("when", System.currentTimeMillis()) .with( "ttl", (System.currentTimeMillis() + TimeUnit.DAYS.toMillis(1L)) / TimeUnit.SECONDS.toMillis(1L) ) ); } return new DyTake(item, this.delay); }
/** * Worst metrics. * @return List of them * @throws IOException If fails */ public Iterable<Iterable<Directive>> worst() throws IOException { return new Mapped<>( item -> new Directives() .add("metric") .attr("id", item.get("metric").getS()) .add("pos").set(item.get("pos").getN()).up() .add("neg").set(item.get("neg").getN()).up() .add("psum").set(new DyNum(item, "psum").doubleValue()).up() .add("pavg").set(new DyNum(item, "pavg").doubleValue()).up() .add("nsum").set(new DyNum(item, "nsum").doubleValue()).up() .add("navg").set(new DyNum(item, "navg").doubleValue()).up() .add("avg").set(new DyNum(item, "avg").doubleValue()).up() .add("champions").set(item.get("champions").getN()).up() .add("artifact").set(item.get("artifact").getS()).up() .add("mean").set(new DyNum(item, "mean").doubleValue()).up() .add("sigma").set(new DyNum(item, "sigma").doubleValue()).up() .up(), this.table.frame() .where("version", new Version().value()) .through( new QueryValve() .withScanIndexForward(false) .withIndexName("mistakes") .withConsistentRead(false) // @checkstyle MagicNumber (1 line) .withLimit(20) .withSelect(Select.ALL_ATTRIBUTES) ) ); }
/** * Public ctor. */ public QueryValve() { this( Tv.TWENTY, true, new ArrayList<String>(0), "", Select.SPECIFIC_ATTRIBUTES.toString(), true ); }
public QueryResult getLatestTweetsForScreenName(String screenName, long timestamp) throws Exception { try { long startDateMilli = System.currentTimeMillis(); Map<String, Condition> keyConditions = new HashMap<String, Condition>(); keyConditions.put( COL_SCREENNAME, new Condition().withComparisonOperator( ComparisonOperator.EQ).withAttributeValueList( new AttributeValue().withS(screenName))); keyConditions.put( COL_CREATEDAT, new Condition().withComparisonOperator( ComparisonOperator.BETWEEN) .withAttributeValueList( new AttributeValue().withN(Long .toString(timestamp)), new AttributeValue().withN(Long .toString(startDateMilli)))); QueryRequest queryRequest = new QueryRequest() .withTableName(TABLE_NAME).withIndexName(INDEX_SCREENNAME) .withKeyConditions(keyConditions) .withSelect(Select.ALL_ATTRIBUTES) .withScanIndexForward(true); QueryResult result = dynamoDB.query(queryRequest); return result; } catch (Exception e) { handleException(e); } return null; }
/** * Add one metric. * @param metric XML with metric * @throws IOException If fails */ private void add(final XML metric) throws IOException { final Item item; final Iterator<Item> items = this.table.frame() .through( new QueryValve() .withLimit(1) .withSelect(Select.ALL_ATTRIBUTES) ) .where("metric", metric.xpath("@name").get(0)) .where("version", new Version().value()) .iterator(); if (items.hasNext()) { item = items.next(); } else { item = this.table.put( new Attributes() .with("metric", metric.xpath("@name").get(0)) .with("version", new Version().value()) .with("artifact", "?") .with("champions", 0L) // @checkstyle MagicNumber (2 lines) .with("mean", new DyNum(0.5d).longValue()) .with("sigma", new DyNum(0.1d).longValue()) ); } final double mean = Double.parseDouble( metric.xpath("mean/text()").get(0) ); final double sigma = Double.parseDouble( metric.xpath("sigma/text()").get(0) ); final boolean reverse = Boolean.parseBoolean( metric.xpath("reverse/text()").get(0) ); final double mbefore = new DyNum(item, "mean").doubleValue(); final double sbefore = new DyNum(item, "sigma").doubleValue(); // @checkstyle BooleanExpressionComplexityCheck (1 line) if (sigma < sbefore || mean < mbefore && reverse || mean > mbefore && !reverse) { item.put( new AttributeUpdates() .with("artifact", metric.xpath("/index/@artifact").get(0)) .with( "champions", new AttributeValueUpdate() .withValue(new AttributeValue().withN("1")) .withAction(AttributeAction.ADD) ) .with("mean", new DyNum(mean).update()) .with("sigma", new DyNum(sigma).update()) ); } }
@Override public int countItems(PageIterator pageIterator) { return toCount(scan(new ScanSpec().withSelect(Select.COUNT), pageIterator)); }
public static void queryIndex(String indexName) { System.out.println ("\n***********************************************************\n"); System.out.print("Querying index " + indexName + "..."); QueryRequest queryRequest = new QueryRequest() .withTableName(tableName) .withIndexName(indexName) .withScanIndexForward(true); HashMap<String, Condition> keyConditions = new HashMap<String, Condition>(); if (indexName == "CreateDateIndex") { System.out.println("Issues filed on 2013-11-01"); keyConditions.put("CreateDate",new Condition() .withComparisonOperator(ComparisonOperator.EQ) .withAttributeValueList(new AttributeValue() .withS("2013-11-01"))); keyConditions.put("IssueId",new Condition() .withComparisonOperator(ComparisonOperator.BEGINS_WITH) .withAttributeValueList(new AttributeValue().withS("A-"))); } else if (indexName == "TitleIndex") { System.out.println("Compilation errors"); keyConditions.put("Title",new Condition() .withComparisonOperator( ComparisonOperator.EQ) .withAttributeValueList(new AttributeValue() .withS("Compilation error"))); keyConditions.put("IssueId", new Condition() .withComparisonOperator(ComparisonOperator.BEGINS_WITH) .withAttributeValueList(new AttributeValue().withS("A-"))); // Select queryRequest.setSelect(Select.ALL_PROJECTED_ATTRIBUTES); } else if (indexName == "DueDateIndex") { System.out.println("Items that are due on 2013-11-30"); keyConditions.put("DueDate",new Condition() .withComparisonOperator(ComparisonOperator.EQ) .withAttributeValueList(new AttributeValue().withS("2013-11-30"))); // Select queryRequest.setSelect(Select.ALL_PROJECTED_ATTRIBUTES); } else { System.out.println("\nNo valid index name provided"); return; } queryRequest.setKeyConditions(keyConditions); QueryResult result = client.query(queryRequest); List<Map<String, AttributeValue>> items = result.getItems(); Iterator<Map<String, AttributeValue>> itemsIter = items.iterator(); System.out.println(); while (itemsIter.hasNext()) { Map<String, AttributeValue> currentItem = itemsIter.next(); Iterator<String> currentItemIter = currentItem.keySet().iterator(); while (currentItemIter.hasNext()) { String attr = (String) currentItemIter.next(); if (attr == "Priority" ) { System.out.println(attr + "---> " + currentItem.get(attr).getN()); } else { System.out.println(attr + "---> " + currentItem.get(attr).getS()); } } System.out.println(); } }
protected QueryRequest buildQueryRequest(String tableName, String theIndexName, String hashKeyAttributeName, String rangeKeyAttributeName, String rangeKeyPropertyName, List<Condition> hashKeyConditions, List<Condition> rangeKeyConditions) { // TODO Set other query request properties based on config QueryRequest queryRequest = new QueryRequest(); queryRequest.setTableName(tableName); queryRequest.setIndexName(theIndexName); if (isApplicableForGlobalSecondaryIndex()) { List<String> allowedSortProperties = new ArrayList<String>(); for (Entry<String, List<Condition>> singlePropertyCondition : propertyConditions.entrySet()) { if (entityInformation.getGlobalSecondaryIndexNamesByPropertyName().keySet() .contains(singlePropertyCondition.getKey())) { allowedSortProperties.add(singlePropertyCondition.getKey()); } } HashMap<String, Condition> keyConditions = new HashMap<String, Condition>(); if (hashKeyConditions != null && hashKeyConditions.size() > 0) { for (Condition hashKeyCondition : hashKeyConditions) { keyConditions.put(hashKeyAttributeName, hashKeyCondition); allowedSortProperties.add(hashKeyPropertyName); } } if (rangeKeyConditions != null && rangeKeyConditions.size() > 0) { for (Condition rangeKeyCondition : rangeKeyConditions) { keyConditions.put(rangeKeyAttributeName, rangeKeyCondition); allowedSortProperties.add(rangeKeyPropertyName); } } for (Entry<String, List<Condition>> singleAttributeConditions : attributeConditions.entrySet()) { for (Condition condition : singleAttributeConditions.getValue()) { keyConditions.put(singleAttributeConditions.getKey(), condition); } } queryRequest.setKeyConditions(keyConditions); queryRequest.setSelect(Select.ALL_PROJECTED_ATTRIBUTES); applySortIfSpecified(queryRequest, new ArrayList<String>(new HashSet<String>(allowedSortProperties))); } return queryRequest; }
@Override public Dosage fetch(final Credentials credentials, final String table, final Map<String, Condition> conditions, final Collection<String> keys) throws IOException { final AmazonDynamoDB aws = credentials.aws(); try { final Collection<String> attrs = new HashSet<String>( Arrays.asList(this.attributes) ); attrs.addAll(keys); QueryRequest request = new QueryRequest() .withTableName(table) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL) .withKeyConditions(conditions) .withConsistentRead(this.consistent) .withScanIndexForward(this.forward) .withSelect(this.select) .withLimit(this.limit); if (this.select.equals(Select.SPECIFIC_ATTRIBUTES.toString())) { request = request.withAttributesToGet(attrs); } if (!this.index.isEmpty()) { request = request.withIndexName(this.index); } final long start = System.currentTimeMillis(); final QueryResult result = aws.query(request); Logger.info( this, "#items(): loaded %d item(s) from '%s' using %s, %s, in %[ms]s", result.getCount(), table, conditions, new PrintableConsumedCapacity( result.getConsumedCapacity() ).print(), System.currentTimeMillis() - start ); return new QueryValve.NextDosage(credentials, request, result); } catch (final AmazonClientException ex) { throw new IOException( String.format( "failed to fetch from \"%s\" by %s and %s", table, conditions, keys ), ex ); } finally { aws.shutdown(); } }
/** * With attributes to select. * @param slct Select to use * @return New query valve * @see QueryRequest#withSelect(Select) * @since 0.10.2 * @checkstyle AvoidDuplicateLiterals (5 line) */ public QueryValve withSelect(final Select slct) { return new QueryValve( this.limit, this.forward, Arrays.asList(this.attributes), this.index, slct.toString(), this.consistent ); }