@Query("sayHello") HelloReply sayHello( HelloRequest request, StreamingGreeterGrpc.StreamingGreeterStub client, DataFetchingEnvironment dataFetchingEnvironment) { client.sayHelloStreaming( request, new GraphQlStreamObserver<HelloReply, GraphQlResponse>(dataFetchingEnvironment) { @Override protected GraphQlResponse getData(HelloReply value, ListValue path) { // TODO: how can this be improved? QueryType data = QueryType.newBuilder() .setHelloReply( io.grpc.examples.graphql.HelloReply.newBuilder() .setMessage(value.getMessage()) .build()) .build(); return GraphQlResponse.newBuilder().setPath(path).setData(data).build(); } }); return null; }
@Override public void doMerge(JsonParser parser, int currentDepth, Message.Builder messageBuilder) throws IOException { Value.Builder builder = (Value.Builder) messageBuilder; JsonToken token = parser.currentToken(); if (token.isBoolean()) { builder.setBoolValue(ParseSupport.parseBool(parser)); } else if (token.isNumeric()) { builder.setNumberValue(ParseSupport.parseDouble(parser)); } else if (token == JsonToken.VALUE_NULL) { builder.setNullValue(NullValue.NULL_VALUE); } else if (token.isScalarValue()) { builder.setStringValue(ParseSupport.parseString(parser)); } else if (token == JsonToken.START_OBJECT) { Struct.Builder structBuilder = builder.getStructValueBuilder(); StructMarshaller.INSTANCE.mergeValue(parser, currentDepth + 1, structBuilder); } else if (token == JsonToken.START_ARRAY) { ListValue.Builder listValueBuilder = builder.getListValueBuilder(); ListValueMarshaller.INSTANCE.mergeValue(parser, currentDepth + 1, listValueBuilder); } else { throw new IllegalStateException("Unexpected json data: " + parser.getText()); } }
@Test public void anyInMaps() throws Exception { TestAny.Builder testAny = TestAny.newBuilder(); testAny.putAnyMap("int32_wrapper", Any.pack(Int32Value.newBuilder().setValue(123).build())); testAny.putAnyMap("int64_wrapper", Any.pack(Int64Value.newBuilder().setValue(456).build())); testAny.putAnyMap("timestamp", Any.pack(Timestamps.parse("1969-12-31T23:59:59Z"))); testAny.putAnyMap("duration", Any.pack(Durations.parse("12345.1s"))); testAny.putAnyMap("field_mask", Any.pack(FieldMaskUtil.fromString("foo.bar,baz"))); Value numberValue = Value.newBuilder().setNumberValue(1.125).build(); Struct.Builder struct = Struct.newBuilder(); struct.putFields("number", numberValue); testAny.putAnyMap("struct", Any.pack(struct.build())); Value nullValue = Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build(); testAny.putAnyMap( "list_value", Any.pack(ListValue.newBuilder().addValues(numberValue).addValues(nullValue).build())); testAny.putAnyMap("number_value", Any.pack(numberValue)); testAny.putAnyMap("any_value_number", Any.pack(Any.pack(numberValue))); testAny.putAnyMap("any_value_default", Any.pack(Any.getDefaultInstance())); testAny.putAnyMap("default", Any.getDefaultInstance()); assertMatchesUpstream(testAny.build(), TestAllTypes.getDefaultInstance()); }
private static boolean parseInlineGenotypeFields(String field, VariantCall.Builder vcBuilder, ListValue.Builder lvBuilder, IntGenotypeFieldAccessors.Accessor accessor, Genotype g) { final int[] intValues = accessor.getValues(g); if (intValues == null || intValues.length == 0) { return false; } if (field.equals(VCFConstants.GENOTYPE_PL_KEY)) { // HTSJDK folds GL's into PL's. We only use PL's to store genotype likelihood. for (int i = 0; i < intValues.length; i++) { // We add 0.0 to remove the possiblity of getting -0.0. vcBuilder.addGenotypeLikelihood(-(double) intValues[i] / 10.0 + 0.0); } return false; } for (int i = 0; i < intValues.length; i++) { lvBuilder.addValues(Value.newBuilder().setNumberValue(intValues[i])); } return true; }
private static VariantCall makeVariantCall( String name, int[] g, double[] gl, boolean isPhased, Map<String, ListValue> info) { VariantCall.Builder vcBuilder = VariantCall.newBuilder().setCallSetName(name); if (g != null) { for (int i = 0; i < g.length; i++) { vcBuilder.addGenotype(g[i]); } if (isPhased && g.length > 1) { vcBuilder.setPhaseset("*"); } } if (gl != null) { for (int i = 0; i < gl.length; i++) { vcBuilder.addGenotypeLikelihood(gl[i]); } } if (info != null) { vcBuilder.getMutableInfo().putAll(info); } return vcBuilder.build(); }
@ProcessElement public void processElement(ProcessContext c) throws Exception { ListValue lv = c.element().getInfo().get("AF"); if (lv != null && lv.getValuesCount() > 0) { Position position = Position.newBuilder() .setPosition(c.element().getStart()) .setReferenceName(c.element().getReferenceName()) .build(); AlleleFreq af = new AlleleFreq(); af.setRefFreq(Double.parseDouble(lv.getValues(0).getStringValue())); af.setAltBases(c.element().getAlternateBasesList()); af.setRefBases(c.element().getReferenceBases()); c.output(KV.of(position, af)); } else { // AF field wasn't populated in info, so we don't have frequency information // for this Variant. // TODO instead of straight throwing an exception, log a warning. If at the end of this // step the number of AlleleFreqs retrieved is below a given threshold, then throw an // exception. throw new IllegalArgumentException("Variant " + c.element().getId() + " does not have " + "allele frequency information stored in INFO field AF."); } }
@Test public void testGetAlleleFreq() throws Exception { DoFnTester<Variant, KV<Position, AlleleFreq>> getAlleleFreq = DoFnTester.of( new GetAlleleFreq()); Position pos = Position.newBuilder() .setReferenceName("1") .setPosition(123L) .build(); Variant.Builder vBuild = Variant.newBuilder() .setReferenceName("1") .setStart(123L) .setReferenceBases("C") .addAlternateBases("T"); vBuild.getMutableInfo().put("AF", ListValue.newBuilder() .addValues(Value.newBuilder().setStringValue("0.25").build()).build()); AlleleFreq af = new AlleleFreq(); af.setAltBases(Lists.newArrayList("T")); af.setRefBases("C"); af.setRefFreq(0.25); Assert.assertThat(getAlleleFreq.processBundle(vBuild.build()), CoreMatchers.hasItems(KV.of(pos, af))); }
@Test public void testJustOneSnp() throws Exception { Variant snp1BiAllelic = TestHelper.makeVariant("chr2", 100, "C", Arrays.asList("A"), "het-RA").build(); Variant snp2BiAllelic = TestHelper.makeVariant("chr2", 200, "G", Arrays.asList("T"), "hom-AA").build(); Map<String, ListValue> info = new HashMap<String, ListValue>(); ListValue.Builder callSetNames = ListValue.newBuilder(); info.put(MergeAllVariantsAtSameSite.OVERLAPPING_CALLSETS_FIELD, callSetNames.build()); Variant expectedOutput1 = Variant.newBuilder(snp1BiAllelic) .putAllInfo(info) .build(); Variant expectedOutput2 = Variant.newBuilder(snp2BiAllelic) .putAllInfo(info) .build(); VariantMergeStrategyTestHelper.mergeTest(100L, Arrays.asList(snp1BiAllelic, snp2BiAllelic), Arrays.asList(expectedOutput1, expectedOutput2), MergeAllVariantsAtSameSite.class); }
@Override public void onNext(T value) { List<Value> path = dataFetchingEnvironment .getFieldTypeInfo() .getPath() .toList() .stream() .map( p -> p instanceof Number ? Value.newBuilder() .setNumberValue(Double.parseDouble(p.toString())) .build() : Value.newBuilder().setStringValue(p.toString()).build()) .collect(ImmutableList.toImmutableList()); ListValue pathListVale = ListValue.newBuilder() .addAllValues(path) .addValues(Value.newBuilder().setNumberValue(pathIndex.incrementAndGet())) .build(); R graphQlResponse = getData(value, pathListVale); rejoinerStreamingContext.responseStreamObserver().onNext(graphQlResponse); try { System.out.println( "Streaming response as Json: " + JsonFormat.printer().print(graphQlResponse)); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } }
@Override public void doMerge(JsonParser parser, int currentDepth, Message.Builder messageBuilder) throws IOException { JsonToken token = parser.currentToken(); if (token != JsonToken.START_ARRAY) { throw new InvalidProtocolBufferException("Expect an array but found: " + parser.getText()); } ListValue.Builder builder = (ListValue.Builder) messageBuilder; while (parser.nextValue() != JsonToken.END_ARRAY) { Value.Builder valueBuilder = builder.addValuesBuilder(); ValueMarshaller.INSTANCE.mergeValue(parser, currentDepth + 1, valueBuilder); } }
@Override public void doWrite(ListValue message, JsonGenerator gen) throws IOException { List<Value> values = message.getValuesList(); int numElements = values.size(); gen.writeStartArray(numElements); for (int i = 0; i < numElements; i++) { ValueMarshaller.INSTANCE.writeValue(values.get(i), gen); } gen.writeEndArray(); }
@Test public void struct() throws Exception { // Build a struct with all possible values. TestStruct.Builder builder = TestStruct.newBuilder(); Struct.Builder structBuilder = builder.getStructValueBuilder(); structBuilder.putFields("null_value", Value.newBuilder().setNullValueValue(0).build()); structBuilder.putFields("number_value", Value.newBuilder().setNumberValue(1.25).build()); structBuilder.putFields("string_value", Value.newBuilder().setStringValue("hello").build()); Struct.Builder subStructBuilder = Struct.newBuilder(); subStructBuilder.putFields("number_value", Value.newBuilder().setNumberValue(1234).build()); structBuilder.putFields( "struct_value", Value.newBuilder().setStructValue(subStructBuilder.build()).build()); ListValue.Builder listBuilder = ListValue.newBuilder(); listBuilder.addValues(Value.newBuilder().setNumberValue(1.125).build()); listBuilder.addValues(Value.newBuilder().setNullValueValue(0).build()); structBuilder.putFields( "list_value", Value.newBuilder().setListValue(listBuilder.build()).build()); TestStruct message = builder.build(); assertMatchesUpstream(message); builder = TestStruct.newBuilder(); builder.setValue(Value.newBuilder().setNullValueValue(0).build()); message = builder.build(); assertMatchesUpstream(message); builder = TestStruct.newBuilder(); listBuilder = builder.getListValueBuilder(); listBuilder.addValues(Value.newBuilder().setNumberValue(31831.125).build()); listBuilder.addValues(Value.newBuilder().setNullValueValue(0).build()); message = builder.build(); assertMatchesUpstream(message); }
public static DefaultData tensorToNDArray(DefaultData data){ if (data.getDataOneofCase() == DataOneofCase.TENSOR) { List<Double> valuesList = data.getTensor().getValuesList(); List<Integer> shapeList = data.getTensor().getShapeList(); DefaultData.Builder dataBuilder = DefaultData.newBuilder(); int index=0; for (Iterator<String> i = data.getNamesList().iterator(); i.hasNext();){ dataBuilder.setNames(index, i.next()); index++; } ListValue.Builder b1 = ListValue.newBuilder(); for (int i = 0; i < shapeList.get(0); ++i) { ListValue.Builder b2 = ListValue.newBuilder(); for (int j = 0; j < shapeList.get(1); j++){ b2.addValues(Value.newBuilder().setNumberValue(valuesList.get(i*shapeList.get(1))+j)); } b1.addValues(Value.newBuilder().setListValue(b2.build())); } dataBuilder.setNdarray(b1.build()); return dataBuilder.build(); } else if (data.getDataOneofCase() == DataOneofCase.NDARRAY) { return data; } return null; }
public static DefaultData updateData(DefaultData oldData, INDArray newData){ DefaultData.Builder dataBuilder = DefaultData.newBuilder(); dataBuilder.addAllNames(oldData.getNamesList()); // int index=0; // for (Iterator<String> i = oldData.getFeaturesList().iterator(); i.hasNext();){ // dataBuilder.setFeatures(index, i.next()); // index++; // } if (oldData.getDataOneofCase() == DataOneofCase.TENSOR){ Tensor.Builder tBuilder = Tensor.newBuilder(); List<Integer> shapeList = Arrays.stream(newData.shape()).boxed().collect(Collectors.toList()); tBuilder.addAllShape(shapeList); for (int i=0; i<shapeList.get(0); ++i){ for (int j=0; j<shapeList.get(1); ++j){ tBuilder.addValues(newData.getDouble(i,j)); } } dataBuilder.setTensor(tBuilder); return dataBuilder.build(); } else if (oldData.getDataOneofCase() == DataOneofCase.NDARRAY){ ListValue.Builder b1 = ListValue.newBuilder(); for (int i = 0; i < newData.shape()[0]; ++i) { ListValue.Builder b2 = ListValue.newBuilder(); for (int j = 0; j < newData.shape()[1]; j++){ b2.addValues(Value.newBuilder().setNumberValue(newData.getDouble(i,j))); } b1.addValues(Value.newBuilder().setListValue(b2.build())); } dataBuilder.setNdarray(b1.build()); return dataBuilder.build(); } return null; }
@Test public void testSimpleNDArrayCase() throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException { List<SeldonMessage> predictorReturns = new ArrayList<>(); String[] names = {"c","d"}; Double[] values1 = {1.0,1.0}; predictorReturns.add(SeldonMessage.newBuilder().setStatus(Status.newBuilder().setStatus(Status.StatusFlag.SUCCESS).build()) .setData(DefaultData.newBuilder().addAllNames(Arrays.asList(names)) .setNdarray(ListValue.newBuilder().addValues(Value.newBuilder().setListValue(ListValue.newBuilder() .addValues(Value.newBuilder().setNumberValue(values1[0])) .addValues(Value.newBuilder().setNumberValue(values1[1])).build())).build()).build()).build()); Double[] values2 = {1.0,0.5}; predictorReturns.add(SeldonMessage.newBuilder().setStatus(Status.newBuilder().setStatus(Status.StatusFlag.SUCCESS).build()) .setData(DefaultData.newBuilder().addAllNames(Arrays.asList(names)) .setNdarray(ListValue.newBuilder().addValues(Value.newBuilder().setListValue(ListValue.newBuilder() .addValues(Value.newBuilder().setNumberValue(values2[0])) .addValues(Value.newBuilder().setNumberValue(values2[1])).build())).build()).build()).build()); Double[] values3 = {2.2,0.9}; predictorReturns.add(SeldonMessage.newBuilder().setStatus(Status.newBuilder().setStatus(Status.StatusFlag.SUCCESS).build()) .setData(DefaultData.newBuilder().addAllNames(Arrays.asList(names)) .setNdarray(ListValue.newBuilder().addValues(Value.newBuilder().setListValue(ListValue.newBuilder() .addValues(Value.newBuilder().setNumberValue(values3[0])) .addValues(Value.newBuilder().setNumberValue(values3[1])).build())).build()).build()).build()); AverageCombinerUnit averageCombinerUnit = new AverageCombinerUnit(); SeldonMessage average = averageCombinerUnit.aggregate(predictorReturns, null); Assert.assertThat(average.getData().getNamesList().get(0),is(names[0])); Double[][] expected_values = {{(1.0+1.0+2.2)/3,(1.0+0.5+0.9)/3}}; Assert.assertEquals(expected_values[0][0],average.getData().getNdarray().getValues(0).getListValue().getValues(0).getNumberValue(),1e-7); }
/** * Creates additional types (Value, Struct and ListValue) to be added to the Service config. * TODO (guptasu): Fix this hack. Find a better way to add the predefined types. * TODO (guptasu): Add them only when required and not in all cases. */ static Iterable<Type> createAdditionalServiceTypes() { Map<String, DescriptorProto> additionalMessages = Maps.newHashMap(); additionalMessages.put(Struct.getDescriptor().getFullName(), Struct.getDescriptor().toProto()); additionalMessages.put(Value.getDescriptor().getFullName(), Value.getDescriptor().toProto()); additionalMessages.put(ListValue.getDescriptor().getFullName(), ListValue.getDescriptor().toProto()); additionalMessages.put(Empty.getDescriptor().getFullName(), Empty.getDescriptor().toProto()); additionalMessages.put(Int32Value.getDescriptor().getFullName(), Int32Value.getDescriptor().toProto()); additionalMessages.put(DoubleValue.getDescriptor().getFullName(), DoubleValue.getDescriptor().toProto()); additionalMessages.put(BoolValue.getDescriptor().getFullName(), BoolValue.getDescriptor().toProto()); additionalMessages.put(StringValue.getDescriptor().getFullName(), StringValue.getDescriptor().toProto()); for (Descriptor descriptor : Struct.getDescriptor().getNestedTypes()) { additionalMessages.put(descriptor.getFullName(), descriptor.toProto()); } // TODO (guptasu): Remove this hard coding. Without this, creation of Model from Service throws. // Needs investigation. String fileName = "struct.proto"; List<Type> additionalTypes = Lists.newArrayList(); for (String typeName : additionalMessages.keySet()) { additionalTypes.add(TypesBuilderFromDescriptor.createType(typeName, additionalMessages.get(typeName), fileName)); } return additionalTypes; }
@VisibleForTesting static Map<String, ListValue> getInfo(VariantContext vc, VCFHeader header) { Map<String, ListValue> toReturn = new HashMap<>(); for (Map.Entry<String, Object> entry : vc.getAttributes().entrySet()) { String currKey = entry.getKey(); VCFInfoHeaderLine metaData = header.getInfoHeaderLine(currKey); // All info fields must have a corresponding header field. if (metaData == null) { logger.log(Level.WARNING, String.format("Could not find matching VCF header field, " + "skipping info field %s", currKey)); continue; } Object currObject = entry.getValue(); ListValue.Builder listValueBuilder = ListValue.newBuilder(); VCFHeaderLineType type = metaData.getType(); if (!(currObject instanceof List)) { toReturn.put(currKey, listValueBuilder.addValues(createTypedValue(type, currObject)).build()); continue; } List<Object> currObjectList = (List<Object>) currObject; for (Object currObj : currObjectList) { listValueBuilder.addValues(createTypedValue(type, currObj)); } toReturn.put(currKey, listValueBuilder.build()); } return toReturn; }
private static ListValue listFromObject(Object obj) { ListValue.Builder lvBuilder = ListValue.newBuilder(); if (!(obj instanceof List)) { lvBuilder.addValues(valueFromObject(obj)); } else { List<Object> objList = (List<Object>) obj; for (int i = 0; i < objList.size(); i++) { lvBuilder.addValues(valueFromObject(objList.get(i))); } } return lvBuilder.build(); }
@Test public void testGetInfo() throws Exception { File vcfFile = new File(VcfToVariant.class.getClassLoader().getResource(VALID_VCF_4_1).getFile()); VCFFileReader vcfReader = new VCFFileReader(vcfFile, false); VCFHeader vcfHeader = vcfReader.getFileHeader(); int currVariant = 0; for (final VariantContext vc : vcfReader) { Map<String, ListValue> info = VcfToVariant.getInfo(vc, vcfHeader); assertEquals(info, TRUTH.get(currVariant).getInfo()); currVariant++; } }
@ProcessElement public void processElement(ProcessContext context) { Variant variant = context.element(); // We may have variants without calls if any callsets were deleted from the variant set. // Skip those. if (null == variant.getCallsList() || variant.getCallsList().isEmpty()) { return; } // Don't filter non-variant segments. if (!filterLowQualityPassing || VariantUtils.IS_NON_VARIANT_SEGMENT.apply(variant)) { context.output(variant); return; } List<VariantCall> filteredCalls = Lists.newArrayList(Iterables.filter(variant.getCallsList(), new Predicate<VariantCall>() { @Override public boolean apply(VariantCall call) { ListValue filters = call.getInfo().get(FILTER_FIELD); if (null != filters && Iterables.any(filters.getValuesList(), isPassing)) { return true; } return false; } })); // After filtering, the variant may no longer have any calls. Skip empty variants. if (filteredCalls.isEmpty()) { return; } context.output(Variant.newBuilder(variant).clearCalls().addAllCalls(filteredCalls).build()); }
@Test public void testFilterVariantCallsFn_AllCallsRemoved() throws Exception { Map<String, ListValue> passingFilter = new HashMap<String, ListValue>(); passingFilter.put( TransformNonVariantSegmentData.FILTER_FIELD, ListValue.newBuilder() .addValues(Value.newBuilder().setStringValue(TransformNonVariantSegmentData.PASSING_FILTER).build()) .build()); VariantCall passingCall = VariantCall.newBuilder().putAllInfo(passingFilter).build(); Map<String, ListValue> failingFilter = new HashMap<String, ListValue>(); failingFilter.put( TransformNonVariantSegmentData.FILTER_FIELD, ListValue.newBuilder() .addValues(Value.newBuilder().setStringValue("VQSRTrancheSNP99.90to100.00").build()) .build()); VariantCall failingCall = VariantCall.newBuilder().putAllInfo(failingFilter).build(); DoFnTester<Variant, Variant> filterCallsFn = DoFnTester.of(new FilterCallsFn(true)); List<Variant> filteredVariants = filterCallsFn.processBundle( Variant.newBuilder().setReferenceBases("T").addAlternateBases("A").addAllCalls(Arrays.asList(failingCall)).build(), Variant.newBuilder().setReferenceBases("G").addAlternateBases("C").addAllCalls(Arrays.asList(passingCall)).build()); assertEquals(1, filteredVariants.size()); // Non-variant segments are not filtered. filteredVariants = filterCallsFn.processBundle( Variant.newBuilder().setReferenceBases("T").addAllCalls(Arrays.asList(failingCall)).build(), Variant.newBuilder().setReferenceBases("G").addAlternateBases(VariantUtils.GATK_NON_VARIANT_SEGMENT_ALT) .addAllCalls(Arrays.asList(failingCall)).build()); assertEquals(2, filteredVariants.size()); }
@Test public void testFormatCalls() throws Exception { Map depthInfo = new HashMap<String, List<String>>(); depthInfo.put( TransformNonVariantSegmentData.DEPTH_FIELD, ListValue.newBuilder() .addValues(Value.newBuilder().setStringValue("30").build()) .build()); Map dotDepthInfo = new HashMap<String, List<String>>(); dotDepthInfo.put( TransformNonVariantSegmentData.DEPTH_FIELD, ListValue.newBuilder() .addValues(Value.newBuilder().setStringValue(".").build()) .build()); VariantCall callWithValidDepth = VariantCall.newBuilder() .setCallSetName("hetAlt").addAllGenotype(Arrays.asList(0, 1)) .putAllInfo(depthInfo) .build(); VariantCall callWithDotDepth = VariantCall.newBuilder() .setCallSetName("homAlt").addAllGenotype(Arrays.asList(1, 1)) .putAllInfo(dotDepthInfo) .build(); Variant variant = Variant.newBuilder() .putAllInfo(FlagVariantsWithAmbiguousCallsFn.NO_AMBIGUOUS_CALLS_INFO) .addAllCalls(Arrays.asList(callWithValidDepth, callWithDotDepth)) .build(); DoFnTester<Variant, TableRow> formatVariantsFn = DoFnTester.of(new TransformNonVariantSegmentData.FormatVariantsFn(true, false, cohortMap)); List<TableRow> rows = formatVariantsFn.processBundle(variant); assertEquals(1, rows.size()); assertEquals("[{call_set_name=hetAlt, phaseset=, genotype=[0, 1], genotype_likelihood=[], FILTER=[], DP=30}," + " {call_set_name=homAlt, phaseset=, genotype=[1, 1], genotype_likelihood=[], FILTER=[], DP=null}]", rows.get(0).get("call").toString()); }
static void noteOverlappingVariantCalls(List<Variant.Builder> currentStartVariantRecords, List<Variant> priorStartVariantRecords) { // Identify the prior variant records overlapping this position. ListValue.Builder priorCallSetNamesBuilder = ListValue.newBuilder(); for (Iterator<Variant> iterator = priorStartVariantRecords.iterator(); iterator.hasNext();) { Variant variantRecord = iterator.next(); if (VariantUtils.isOverlapping(variantRecord, currentStartVariantRecords.get(0))) { priorCallSetNamesBuilder.addAllValues(Lists.transform(variantRecord.getCallsList(), GET_CALL_SET_NAME)); } else if (variantRecord.getStart() < currentStartVariantRecords.get(0).getStart()) { // Remove the current element from the iterator and the list since it is // left of this position. iterator.remove(); } else { // Break out of this loop since we are now examining records to the right of this position. break; } } ListValue priorCallSetNames = priorCallSetNamesBuilder.build(); for (Variant.Builder destVariant : currentStartVariantRecords) { ListValue.Builder overlappingCallSetNames = ListValue.newBuilder(priorCallSetNames); // Also include the call sets for other variants at this same position to each other. for (Variant.Builder srcVariant: currentStartVariantRecords) { if (destVariant.equals(srcVariant)) { continue; } overlappingCallSetNames.addAllValues(Lists.transform(srcVariant.getCallsList(), GET_CALL_SET_NAME)); } Map<String, ListValue> info = new HashMap<String, ListValue>(); info.put(OVERLAPPING_CALLSETS_FIELD, overlappingCallSetNames.build()); destVariant.putAllInfo(info); } }
/** * Grab the value of the provided SAM tag if it exists (returns null otherwise) * @param read the read whose SAM tags need to be searched * @param tag the desired tag * @return string corresponding to SAM tag or null if it doesn't exist */ public static String getSamTag(Read read, String tag) { ListValue value = read.getInfo().get(tag); if (value == null) { return null; } return value.getValues(0).getStringValue(); }
@Test public void testCombineSnpsSameSite() throws Exception { Variant snp1BiAllelic = TestHelper.makeVariant("chr2", 100, "C", Arrays.asList("A"), "het-RA").build(); Variant snp2BiAllelic = TestHelper.makeVariant("chr2", 100, "C", Arrays.asList("G"), "hom-AA").build(); Variant snp3BiAllelic = TestHelper.makeVariant("chr2", 100, "C", Arrays.asList("T"), "het-RA").build(); Variant snp1MultiAllelic = TestHelper.makeVariant("chr2", 100, "C", Arrays.asList("A","G"), "het-AA").build(); Variant snp2MultiAllelic = TestHelper.makeVariant("chr2", 100, "C", Arrays.asList("A","T"), "het-AA").build(); Variant snp3MultiAllelic = TestHelper.makeVariant("chr2", 100, "C", Arrays.asList("G","T"), "het-AA").build(); List<Variant> input = Arrays.asList(snp1BiAllelic, snp2BiAllelic, snp3BiAllelic, snp1MultiAllelic, snp2MultiAllelic, snp3MultiAllelic); Map<String, ListValue> info = new HashMap<String, ListValue>(); ListValue.Builder callSetNames = ListValue.newBuilder(); info.put(MergeAllVariantsAtSameSite.OVERLAPPING_CALLSETS_FIELD, callSetNames.build()); Variant expectedOutput = TestHelper.makeVariant("chr2", 100, 101, "C", Arrays.asList("A", "G", "T")) .addCalls(TestHelper.makeCall("het-RA-[A]", 0, 1)) .addCalls(TestHelper.makeCall("het-AA-[A, G]", 1, 2)) .addCalls(TestHelper.makeCall("het-AA-[A, T]", 1, 3)) .addCalls(TestHelper.makeCall("hom-AA-[G]", 2, 2)) .addCalls(TestHelper.makeCall("het-AA-[G, T]", 2, 3)) .addCalls(TestHelper.makeCall("het-RA-[T]", 0, 3)) .putAllInfo(info) .build(); VariantMergeStrategyTestHelper.mergeTest(100L, input, Arrays.asList(expectedOutput), MergeAllVariantsAtSameSite.class); // Ensure that the result is stable regardless of the input order of these Variants. for (int i = 0; i < 5; i++) { Collections.shuffle(input); VariantMergeStrategyTestHelper.mergeTest(100L, input, Arrays.asList(expectedOutput), MergeAllVariantsAtSameSite.class); } }
@Test public void testMerge() throws Exception { ImmutableList<StreamVariantsRequest> requests = ShardUtils.getVariantRequests(PROTOTYPE, 100L, "chr17:41198773:41198774"); assertEquals(1, requests.size()); Iterator<StreamVariantsResponse> iter = VariantStreamIterator.enforceShardBoundary(IntegrationTestHelper.getAuthFromApplicationDefaultCredential(), requests.get(0), ShardBoundary.Requirement.OVERLAPS, "variants(alternateBases,calls(callSetName,genotype),end,referenceBases,referenceName,start)"); Map<String, ListValue> emptyInfo = new HashMap<String, ListValue>(); emptyInfo.put(MergeAllVariantsAtSameSite.OVERLAPPING_CALLSETS_FIELD, ListValue.newBuilder().build()); // Platinum genomes has both a snp and an insertion at this genomic site // but merging the calls together yields an ambiguous result. Variant expectedOutput1 = TestHelper.makeVariant("chr17", 41198773, 41198774, "C", Arrays.asList("A", "CA")) .addCalls(TestHelper.makeCall("NA12878", 0, 1)) // ambiguous .addCalls(TestHelper.makeCall("NA12892", 0, 2)) // ambiguous .addCalls(TestHelper.makeCall("NA12878", 0, 2)) // ambiguous .addCalls(TestHelper.makeCall("NA12877", 0, 0)) .addCalls(TestHelper.makeCall("NA12889", 0, 0)) .addCalls(TestHelper.makeCall("NA12891", 0, 0)) .addCalls(TestHelper.makeCall("NA12892", 0, 0)) // ambiguous .addCalls(TestHelper.makeCall("NA12890", 0, 0)) .putAllInfo(emptyInfo) .build(); VariantMergeStrategyTestHelper.mergeTest(requests.get(0).getStart(), iter.next().getVariantsList(), Arrays.asList(expectedOutput1), MergeAllVariantsAtSameSite.class); }
ListValueMarshaller() { super(ListValue.getDefaultInstance()); }
private static boolean parseOtherGenotypeFields(String field, VariantContext vc, ListValue.Builder lvBuilder, Genotype g, VCFHeader header) { if (!g.hasAnyAttribute(field)) { return false; } final VCFFormatHeaderLine metaData = header.getFormatHeaderLine(field); if (metaData == null) { logger.log(Level.WARNING, String.format("Could not find matching VCF header field for " + "genotype field %s", field)); return false; } VCFHeaderLineType type = metaData.getType(); Object value = g.getExtendedAttribute(field); final int fieldCount = metaData.getCount(vc); if (fieldCount == 1) { lvBuilder.addValues(createTypedValue(type, value)); return true; } if (!(value instanceof String)) { throw new IllegalStateException("received non-Flag genotype field as non-String type"); } String[] valueArray = ((String) value).split(","); if (valueArray.length == 1) { throw new IllegalStateException(String.format("header indicating a count greater than 1 " + "with non-List type found for field %s", field)); } boolean allFalse = true; for (int i = 0; i < valueArray.length; i++) { VCFHeaderLineType thisType = VCFHeaderLineType.String; if (!valueArray[i].equals(VCFConstants.MISSING_VALUE_v4)) { thisType = type; allFalse = false; } lvBuilder.addValues(createTypedValue(thisType, valueArray[i])); } // We only add the lvBuilder if there is at least one non-missing value return !allFalse; }
@VisibleForTesting static List<VariantCall> getCalls(VariantContext vc, VCFHeader header) { List<VariantCall> toReturn = new ArrayList<>(); for (String currSample : header.getGenotypeSamples()) { if (!vc.hasGenotype(currSample)) { continue; } Genotype currGenotype = vc.getGenotype(currSample); VariantCall.Builder vcBuilder = VariantCall.newBuilder(); vcBuilder.setCallSetName(currSample); // Get GT info. final Map<Allele, Integer> alleleStrings = buildAlleleMap(vc); vcBuilder.addGenotype(alleleStrings.get(currGenotype.getAllele(0))); for (int i = 1; i < currGenotype.getPloidy(); i++) { vcBuilder.addGenotype(alleleStrings.get(currGenotype.getAllele(i))); } // Set phasing (not applicable to haploid). if (currGenotype.isPhased() && currGenotype.getPloidy() > 1) { vcBuilder.setPhaseset("*"); } // Get rest of the genotype info. Map<String, ListValue> genotypeInfo = new HashMap<>(); // Set filters if (currGenotype.isFiltered()) { genotypeInfo.put(VCFConstants.GENOTYPE_FILTER_KEY, ListValue.newBuilder() .addValues(Value.newBuilder().setStringValue(currGenotype.getFilters()).build()) .build()); } for (final String field : vc.calcVCFGenotypeKeys(header)) { // We've already handled genotype if (field.equals(VCFConstants.GENOTYPE_KEY)) { continue; } ListValue.Builder listValueBuilder = ListValue.newBuilder(); if (field.equals(VCFConstants.GENOTYPE_FILTER_KEY)) { // This field has already been dealt with continue; } else { final IntGenotypeFieldAccessors.Accessor accessor = GENOTYPE_FIELD_ACCESSORS.getAccessor(field); if (accessor != null) { // The field is a default inline field. if (!parseInlineGenotypeFields(field, vcBuilder, listValueBuilder, accessor, currGenotype)) { continue; } } else { // Other field, we'll get type/other info from header. if (!parseOtherGenotypeFields(field, vc, listValueBuilder, currGenotype, header)) { continue; } } } genotypeInfo.put(field, listValueBuilder.build()); } vcBuilder.putAllInfo(genotypeInfo); toReturn.add(vcBuilder.build()); } return toReturn; }
public static List<Variant> convertContextToVariant(Iterable<VariantContext> variantContexts, VCFHeader header) { List<Variant> finalList = new ArrayList<>(); for (final VariantContext vc : variantContexts) { String referenceName = vc.getChr(); // htsjdk start/end are both 1-based closed long variantStart = vc.getStart() - 1; long variantEnd = vc.getEnd(); String refBases = vc.getReference().getBaseString(); boolean hasQuality = vc.hasLog10PError(); double quality = vc.getPhredScaledQual(); boolean hasId = vc.hasID(); String[] ids = vc.getID().split(","); List<String> altBases = getAltBases(vc); List<String> filters = getFilter(vc); Map<String, ListValue> info = getInfo(vc, header); List<VariantCall> calls = getCalls(vc, header); // Actually build the variant Variant.Builder variantBuilder = Variant.newBuilder().setReferenceName(referenceName). setStart(variantStart).setEnd(variantEnd).setReferenceBases(refBases); if (hasQuality) { variantBuilder.setQuality(quality); } if (hasId) { for (String id : ids) { variantBuilder.addNames(id); } } for (String base : altBases) { variantBuilder.addAlternateBases(base); } for (VariantCall call : calls) { variantBuilder.addCalls(call); } for (String filter : filters) { variantBuilder.addFilter(filter); } variantBuilder.putAllInfo(info); finalList.add(variantBuilder.build()); } return finalList; }
public void testStruct() throws Exception { // Build a struct with all possible values. TestStruct.Builder builder = TestStruct.newBuilder(); Struct.Builder structBuilder = builder.getStructValueBuilder(); structBuilder.putFields("null_value", Value.newBuilder().setNullValueValue(0).build()); structBuilder.putFields("number_value", Value.newBuilder().setNumberValue(1.25).build()); structBuilder.putFields("string_value", Value.newBuilder().setStringValue("hello").build()); Struct.Builder subStructBuilder = Struct.newBuilder(); subStructBuilder.putFields("number_value", Value.newBuilder().setNumberValue(1234).build()); structBuilder.putFields( "struct_value", Value.newBuilder().setStructValue(subStructBuilder.build()).build()); ListValue.Builder listBuilder = ListValue.newBuilder(); listBuilder.addValues(Value.newBuilder().setNumberValue(1.125).build()); listBuilder.addValues(Value.newBuilder().setNullValueValue(0).build()); structBuilder.putFields( "list_value", Value.newBuilder().setListValue(listBuilder.build()).build()); TestStruct message = builder.build(); assertEquals( "{\n" + " \"structValue\": {\n" + " \"null_value\": null,\n" + " \"number_value\": 1.25,\n" + " \"string_value\": \"hello\",\n" + " \"struct_value\": {\n" + " \"number_value\": 1234.0\n" + " },\n" + " \"list_value\": [1.125, null]\n" + " }\n" + "}", toJsonString(message)); assertRoundTripEquals(message); builder = TestStruct.newBuilder(); builder.setValue(Value.newBuilder().setNullValueValue(0).build()); message = builder.build(); assertEquals("{\n" + " \"value\": null\n" + "}", toJsonString(message)); assertRoundTripEquals(message); builder = TestStruct.newBuilder(); listBuilder = builder.getListValueBuilder(); listBuilder.addValues(Value.newBuilder().setNumberValue(31831.125).build()); listBuilder.addValues(Value.newBuilder().setNullValueValue(0).build()); message = builder.build(); assertEquals("{\n" + " \"listValue\": [31831.125, null]\n" + "}", toJsonString(message)); assertRoundTripEquals(message); }
public void testAnyInMaps() throws Exception { JsonFormat.TypeRegistry registry = JsonFormat.TypeRegistry.newBuilder().add(TestAllTypes.getDescriptor()).build(); JsonFormat.Printer printer = JsonFormat.printer().usingTypeRegistry(registry); TestAny.Builder testAny = TestAny.newBuilder(); testAny.putAnyMap("int32_wrapper", Any.pack(Int32Value.newBuilder().setValue(123).build())); testAny.putAnyMap("int64_wrapper", Any.pack(Int64Value.newBuilder().setValue(456).build())); testAny.putAnyMap("timestamp", Any.pack(Timestamps.parse("1969-12-31T23:59:59Z"))); testAny.putAnyMap("duration", Any.pack(Durations.parse("12345.1s"))); testAny.putAnyMap("field_mask", Any.pack(FieldMaskUtil.fromString("foo.bar,baz"))); Value numberValue = Value.newBuilder().setNumberValue(1.125).build(); Struct.Builder struct = Struct.newBuilder(); struct.putFields("number", numberValue); testAny.putAnyMap("struct", Any.pack(struct.build())); Value nullValue = Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build(); testAny.putAnyMap( "list_value", Any.pack(ListValue.newBuilder().addValues(numberValue).addValues(nullValue).build())); testAny.putAnyMap("number_value", Any.pack(numberValue)); testAny.putAnyMap("any_value_number", Any.pack(Any.pack(numberValue))); testAny.putAnyMap("any_value_default", Any.pack(Any.getDefaultInstance())); testAny.putAnyMap("default", Any.getDefaultInstance()); assertEquals( "{\n" + " \"anyMap\": {\n" + " \"int32_wrapper\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Int32Value\",\n" + " \"value\": 123\n" + " },\n" + " \"int64_wrapper\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Int64Value\",\n" + " \"value\": \"456\"\n" + " },\n" + " \"timestamp\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Timestamp\",\n" + " \"value\": \"1969-12-31T23:59:59Z\"\n" + " },\n" + " \"duration\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n" + " \"value\": \"12345.100s\"\n" + " },\n" + " \"field_mask\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.FieldMask\",\n" + " \"value\": \"foo.bar,baz\"\n" + " },\n" + " \"struct\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Struct\",\n" + " \"value\": {\n" + " \"number\": 1.125\n" + " }\n" + " },\n" + " \"list_value\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.ListValue\",\n" + " \"value\": [1.125, null]\n" + " },\n" + " \"number_value\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Value\",\n" + " \"value\": 1.125\n" + " },\n" + " \"any_value_number\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Any\",\n" + " \"value\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Value\",\n" + " \"value\": 1.125\n" + " }\n" + " },\n" + " \"any_value_default\": {\n" + " \"@type\": \"type.googleapis.com/google.protobuf.Any\",\n" + " \"value\": {}\n" + " },\n" + " \"default\": {}\n" + " }\n" + "}", printer.print(testAny.build())); assertRoundTripEquals(testAny.build(), registry); }
@Test public void testFilterVariantCallsFn() throws Exception { DoFnTester<Variant, Variant> filterCallsFn = DoFnTester.of(new FilterCallsFn(true)); Map<String, ListValue> passingFilter = new HashMap<String, ListValue>(); passingFilter.put( TransformNonVariantSegmentData.FILTER_FIELD, ListValue.newBuilder() .addValues(Value.newBuilder().setStringValue(TransformNonVariantSegmentData.PASSING_FILTER).build()) .build()); VariantCall call1 = VariantCall.newBuilder().putAllInfo(passingFilter).build(); Map<String, ListValue> failingFilter = new HashMap<String, ListValue>(); failingFilter.put( TransformNonVariantSegmentData.FILTER_FIELD, ListValue.newBuilder() .addValues(Value.newBuilder().setStringValue("VQSRTrancheSNP99.90to100.00").build()) .build()); VariantCall call2 = VariantCall.newBuilder().putAllInfo(failingFilter).build(); Map<String, ListValue> ambiguousFilter = new HashMap<String, ListValue>(); ambiguousFilter.put( TransformNonVariantSegmentData.FILTER_FIELD, ListValue.newBuilder() .addValues(Value.newBuilder().setStringValue("VQSRTrancheSNP99.90to100.00").build()) .addValues(Value.newBuilder().setStringValue(TransformNonVariantSegmentData.PASSING_FILTER).build()) .build()); VariantCall call3 = VariantCall.newBuilder().putAllInfo(ambiguousFilter).build(); // Test a variant. Variant inputVariant = Variant.newBuilder() .setReferenceBases("A") .addAlternateBases("T") .addAllCalls(Arrays.asList(call1, call2, call3)) .build(); Variant expectedVariant = Variant.newBuilder() .setReferenceBases("A") .addAlternateBases("T") .addAllCalls(Arrays.asList(call1, call3)) .build(); Iterator<Variant> filtered1 = filterCallsFn.processBundle(inputVariant).iterator(); assertEquals(filtered1.next(), expectedVariant); assertFalse(filtered1.hasNext()); // Also test a non-variant segment. These are not filtered. Variant inputBlockRecord = Variant.newBuilder() .setReferenceBases("A") .addAllCalls(Arrays.asList(call1, call2, call3)) .build(); Variant expectedBlockRecord = Variant.newBuilder() .setReferenceBases("A") .addAllCalls(Arrays.asList(call1, call2, call3)) .build(); Iterator<Variant> filtered2 = filterCallsFn.processBundle(inputBlockRecord).iterator(); assertEquals(filtered2.next(), expectedBlockRecord); assertFalse(filtered2.hasNext()); }
@Test public void testAmbiguousOverlappingVariantCallsFn() throws Exception { DoFnTester<Variant, Variant> flagVariantsFn = DoFnTester.of(new TransformNonVariantSegmentData.FlagVariantsWithAmbiguousCallsFn()); DoFnTester<Variant, TableRow> formatVariantsFn = DoFnTester.of(new TransformNonVariantSegmentData.FormatVariantsFn(true, false, cohortMap)); VariantCall call1 = VariantCall.newBuilder().setCallSetName("sample1").addAllGenotype(Arrays.asList(0, 1)) .build(); VariantCall call2 = VariantCall.newBuilder().setCallSetName("sample2").addAllGenotype(Arrays.asList(0, 1)) .build(); Map<String, ListValue> info = new HashMap<String, ListValue>(); info.put(MergeAllVariantsAtSameSite.OVERLAPPING_CALLSETS_FIELD, ListValue.newBuilder().addValues(Value.newBuilder().setStringValue("sampleN")).build()); Variant inputVariant = Variant.newBuilder() .addAllCalls(Arrays.asList(call1, call2)) .putAllInfo(info) .build(); Map<String, ListValue> ambiguousInfo = new HashMap<String, ListValue>(); ambiguousInfo.put(MergeAllVariantsAtSameSite.OVERLAPPING_CALLSETS_FIELD, ListValue.newBuilder().addValues(Value.newBuilder().setStringValue("sample2")).build()); Variant ambiguousInputVariant = Variant.newBuilder() .addAllCalls(Arrays.asList(call1, call2)) .putAllInfo(ambiguousInfo) .build(); Map<String, ListValue> expectedInfo = new HashMap<String, ListValue>(); expectedInfo.put(MergeAllVariantsAtSameSite.OVERLAPPING_CALLSETS_FIELD, ListValue.newBuilder().addValues(Value.newBuilder().setStringValue("sampleN")).build()); expectedInfo.put( TransformNonVariantSegmentData.HAS_AMBIGUOUS_CALLS_FIELD, ListValue.newBuilder() .addValues(Value.newBuilder().setStringValue(Boolean.toString(Boolean.FALSE)).build()) .build()); Variant expectedVariant = Variant.newBuilder() .addAllCalls(Arrays.asList(call1, call2)) .putAllInfo(expectedInfo) .build(); Map<String, ListValue> expectedAmbiguousInfo = new HashMap<String, ListValue>(); expectedAmbiguousInfo.put(MergeAllVariantsAtSameSite.OVERLAPPING_CALLSETS_FIELD, ListValue.newBuilder().addValues(Value.newBuilder().setStringValue("sample2")).build()); expectedAmbiguousInfo.put( TransformNonVariantSegmentData.HAS_AMBIGUOUS_CALLS_FIELD, ListValue.newBuilder() .addValues(Value.newBuilder().setStringValue(Boolean.toString(Boolean.TRUE)).build()) .build()); Variant ambiguousExpectedVariant = Variant.newBuilder() .addAllCalls(Arrays.asList(call1, call2)) .putAllInfo(expectedAmbiguousInfo) .build(); List<Variant> flaggedVariants = flagVariantsFn.processBundle(inputVariant, ambiguousInputVariant); assertEquals(expectedVariant, flaggedVariants.get(0)); assertEquals(ambiguousExpectedVariant, flaggedVariants.get(1)); List<TableRow> rows = formatVariantsFn.processBundle(flaggedVariants); assertEquals(2, rows.size()); assertEquals("false", rows.get(0).get(TransformNonVariantSegmentData.HAS_AMBIGUOUS_CALLS_FIELD).toString()); assertEquals("true", rows.get(1).get(TransformNonVariantSegmentData.HAS_AMBIGUOUS_CALLS_FIELD).toString()); assertEquals("[sampleN]", rows.get(0).get(TransformNonVariantSegmentData.OVERLAPPING_CALLSETS_FIELD).toString()); assertEquals("[sample2]", rows.get(1).get(TransformNonVariantSegmentData.OVERLAPPING_CALLSETS_FIELD).toString()); }
private void testGetReferenceSequenceHelper(final String seq, final String cigar, final String md, final String expectedReference) throws IOException { LinearAlignment.Builder alignment = LinearAlignment.newBuilder(); Cigar cigars = TextCigarCodec.decode(cigar); for (int i = 0; i < cigars.numCigarElements(); i++) { CigarElement c = cigars.getCigarElement(i); CigarUnit.Builder unit = CigarUnit.newBuilder().setOperationLength(c.getLength()); switch (c.getOperator()) { case M: unit.setOperation(CigarUnit.Operation.ALIGNMENT_MATCH); break; case I: unit.setOperation(CigarUnit.Operation.INSERT); break; case D: unit.setOperation(CigarUnit.Operation.DELETE); break; case N: unit.setOperation(CigarUnit.Operation.SKIP); break; case S: unit.setOperation(CigarUnit.Operation.CLIP_SOFT); break; case H: unit.setOperation(CigarUnit.Operation.CLIP_HARD); break; case P: unit.setOperation(CigarUnit.Operation.PAD); break; case EQ: unit.setOperation(CigarUnit.Operation.SEQUENCE_MATCH); break; case X: unit.setOperation(CigarUnit.Operation.SEQUENCE_MISMATCH); break; } alignment.addCigar(unit.build()); } final Read.Builder rec = Read.newBuilder() .setFragmentName("test") .setAlignedSequence(seq) .setAlignment(alignment.build()); rec.getMutableInfo().put("MD", ListValue.newBuilder().addValues(0, Value.newBuilder().setStringValue(md).build()).build()); final String refBases = ReadUtils.inferReferenceSequenceByParsingMdFlag(rec.build()); assertEquals(refBases, expectedReference); }
protected abstract R getData(T value, ListValue path);