@Override public void init() throws IOException { registerKey(NullWritable.class.getName(), NullWritableSerializer.class); registerKey(Text.class.getName(), TextSerializer.class); registerKey(LongWritable.class.getName(), LongWritableSerializer.class); registerKey(IntWritable.class.getName(), IntWritableSerializer.class); registerKey(Writable.class.getName(), DefaultSerializer.class); registerKey(BytesWritable.class.getName(), BytesWritableSerializer.class); registerKey(BooleanWritable.class.getName(), BoolWritableSerializer.class); registerKey(ByteWritable.class.getName(), ByteWritableSerializer.class); registerKey(FloatWritable.class.getName(), FloatWritableSerializer.class); registerKey(DoubleWritable.class.getName(), DoubleWritableSerializer.class); registerKey(VIntWritable.class.getName(), VIntWritableSerializer.class); registerKey(VLongWritable.class.getName(), VLongWritableSerializer.class); LOG.info("Hadoop platform inited"); }
@Test public void tryOne() throws IOException, InterruptedException { Text inKey=new Text("qr"); Iterable<VIntWritable> inValues=new RecyclingIterable( VIntWritable.class, new VIntWritable(76412), new VIntWritable(2), new VIntWritable(7) ); reducer.reduce(inKey,inValues,context); Text outKey=new Text("1951-06 qr"); Text outValue=new Text("3 76421"); verify(context).write(outKey,outValue); }
@Override public Vertex<Text, VIntWritable, Text, Writable> getCurrentVertex() throws IOException, InterruptedException { Vertex<Text, VIntWritable, Text, Writable> vertex; String line = getRecordReader().getCurrentValue().toString(); //Parse each line and create the vertex and edges String[] tokens = line.toString().trim().split("\t"); if (tokens.length == 1) { //Null sender on the left hand side vertex = null; throw new IllegalArgumentException("bad data in line: " + line); } else if (tokens.length != 2) { throw new IllegalArgumentException("bad arguments in line: " + line); } else { vertex = getConf().createVertex(); List<Edge<Text, Text>> edges = edgesFrom(tokens); Text vertexId = new Text(tokens[0]); VIntWritable vertexValue = new VIntWritable(0); vertex.initialize(vertexId, vertexValue, edges); } return vertex; }
/** * The actual algorithm. See class documentation for description. */ @Override public void compute(Iterable<Text> messages) throws IOException { try { // Check to see if we received any messages from nodes notifying // that they have only a single edge for (Text incomingMessage : messages) { Text vertex = new Text(incomingMessage.toString().split(":")[0]); int value = Integer.parseInt(incomingMessage.toString().split(":")[1]); setValue(new VIntWritable(getValue().get() + 1 + value)); // Remove the vertex and its corresponding edge removeVertexRequest(vertex); removeEdges(vertex); // System.err.println("Removing: " + vertex.toString()); } // Broadcast the edges if we only have a single edge sendEdges(); } catch (Exception e) { System.err.println(e.toString()); } }
public void write(Writable w) throws IOException { if (w instanceof TypedBytesWritable) { writeTypedBytes((TypedBytesWritable) w); } else if (w instanceof BytesWritable) { writeBytes((BytesWritable) w); } else if (w instanceof ByteWritable) { writeByte((ByteWritable) w); } else if (w instanceof BooleanWritable) { writeBoolean((BooleanWritable) w); } else if (w instanceof IntWritable) { writeInt((IntWritable) w); } else if (w instanceof VIntWritable) { writeVInt((VIntWritable) w); } else if (w instanceof LongWritable) { writeLong((LongWritable) w); } else if (w instanceof VLongWritable) { writeVLong((VLongWritable) w); } else if (w instanceof FloatWritable) { writeFloat((FloatWritable) w); } else if (w instanceof DoubleWritable) { writeDouble((DoubleWritable) w); } else if (w instanceof Text) { writeText((Text) w); } else if (w instanceof ArrayWritable) { writeArray((ArrayWritable) w); } else if (w instanceof MapWritable) { writeMap((MapWritable) w); } else if (w instanceof SortedMapWritable) { writeSortedMap((SortedMapWritable) w); } else if (w instanceof Record) { writeRecord((Record) w); } else { writeWritable(w); // last resort } }
public Class<? extends Writable> readType() throws IOException { Type type = in.readType(); if (type == null) { return null; } switch (type) { case BYTES: return BytesWritable.class; case BYTE: return ByteWritable.class; case BOOL: return BooleanWritable.class; case INT: return VIntWritable.class; case LONG: return VLongWritable.class; case FLOAT: return FloatWritable.class; case DOUBLE: return DoubleWritable.class; case STRING: return Text.class; case VECTOR: return ArrayWritable.class; case MAP: return MapWritable.class; case WRITABLE: return Writable.class; default: throw new RuntimeException("unknown type"); } }
public static void updateObject(Writable obj, byte[] seed) { if (obj instanceof IntWritable) { ((IntWritable)obj).set(Ints.fromByteArray(seed)); } else if (obj instanceof FloatWritable) { ((FloatWritable)obj).set(r.nextFloat()); } else if (obj instanceof DoubleWritable) { ((DoubleWritable)obj).set(r.nextDouble()); } else if (obj instanceof LongWritable) { ((LongWritable)obj).set(Longs.fromByteArray(seed)); } else if (obj instanceof VIntWritable) { ((VIntWritable)obj).set(Ints.fromByteArray(seed)); } else if (obj instanceof VLongWritable) { ((VLongWritable)obj).set(Longs.fromByteArray(seed)); } else if (obj instanceof BooleanWritable) { ((BooleanWritable)obj).set(seed[0] % 2 == 1 ? true : false); } else if (obj instanceof Text) { ((Text)obj).set(BytesUtil.toStringBinary(seed)); } else if (obj instanceof ByteWritable) { ((ByteWritable)obj).set(seed.length > 0 ? seed[0] : 0); } else if (obj instanceof BytesWritable) { ((BytesWritable)obj).set(seed, 0, seed.length); } else if (obj instanceof UTF8) { ((UTF8)obj).set(BytesUtil.toStringBinary(seed)); } else if (obj instanceof MockValueClass) { ((MockValueClass)obj).set(seed); } else { throw new IllegalArgumentException("unknown writable: " + obj.getClass().getName()); } }
public static <VTYPE> byte[] toBytes(VTYPE obj) { final String className = obj.getClass().getName(); if (className.equals(IntWritable.class.getName())) { return Ints.toByteArray(((IntWritable) obj).get()); } else if (className.equals(FloatWritable.class.getName())) { return BytesUtil.toBytes(((FloatWritable) obj).get()); } else if (className.equals(DoubleWritable.class.getName())) { return BytesUtil.toBytes(((DoubleWritable) obj).get()); } else if (className.equals(LongWritable.class.getName())) { return Longs.toByteArray(((LongWritable) obj).get()); } else if (className.equals(VIntWritable.class.getName())) { return Ints.toByteArray(((VIntWritable) obj).get()); } else if (className.equals(VLongWritable.class.getName())) { return Longs.toByteArray(((VLongWritable) obj).get()); } else if (className.equals(BooleanWritable.class.getName())) { return BytesUtil.toBytes(((BooleanWritable) obj).get()); } else if (className.equals(Text.class.getName())) { return ((Text)obj).copyBytes(); } else if (className.equals(ByteWritable.class.getName())) { return Ints.toByteArray((int) ((ByteWritable) obj).get()); } else if (className.equals(BytesWritable.class.getName())) { // TODO: copyBytes instead? return ((BytesWritable) obj).getBytes(); } else { return new byte[0]; } }
public void write(Writable w) throws IOException { if (w instanceof TypedBytesWritable) { writeTypedBytes((TypedBytesWritable) w); } else if (w instanceof BytesWritable) { writeBytes((BytesWritable) w); } else if (w instanceof ByteWritable) { writeByte((ByteWritable) w); } else if (w instanceof BooleanWritable) { writeBoolean((BooleanWritable) w); } else if (w instanceof IntWritable) { writeInt((IntWritable) w); } else if (w instanceof VIntWritable) { writeVInt((VIntWritable) w); } else if (w instanceof LongWritable) { writeLong((LongWritable) w); } else if (w instanceof VLongWritable) { writeVLong((VLongWritable) w); } else if (w instanceof FloatWritable) { writeFloat((FloatWritable) w); } else if (w instanceof DoubleWritable) { writeDouble((DoubleWritable) w); } else if (w instanceof Text) { writeText((Text) w); } else if (w instanceof ArrayWritable) { writeArray((ArrayWritable) w); } else if (w instanceof MapWritable) { writeMap((MapWritable) w); } else if (w instanceof SortedMapWritable) { writeSortedMap((SortedMapWritable<?>) w); } else if (w instanceof Record) { writeRecord((Record) w); } else { writeWritable(w); // last resort } }
/** * Create new XdmValue from value type and Writables. * */ public static XdmValue newValue(ValueType valueType, Object value) { if (value instanceof Text) { return ValueFactory.newValue(valueType, ((Text)value).toString()); } else if (value instanceof BytesWritable) { return ValueFactory.newValue(valueType, ((BytesWritable)value).getBytes()); } else if (value instanceof IntWritable) { return ValueFactory.newValue(valueType, ((IntWritable)value).get()); } else if (value instanceof LongWritable) { return ValueFactory.newValue(valueType, ((LongWritable)value).get()); } else if (value instanceof VIntWritable) { return ValueFactory.newValue(valueType, ((VIntWritable)value).get()); } else if (value instanceof VLongWritable) { return ValueFactory.newValue(valueType, ((VLongWritable)value).get()); } else if (value instanceof BooleanWritable) { return ValueFactory.newValue(valueType, ((BooleanWritable)value).get()); } else if (value instanceof FloatWritable) { return ValueFactory.newValue(valueType, ((FloatWritable)value).get()); } else if (value instanceof DoubleWritable) { return ValueFactory.newValue(valueType, ((DoubleWritable)value).get()); } else if (value instanceof MarkLogicNode) { return ValueFactory.newValue(valueType, ((MarkLogicNode)value).get()); } else { throw new UnsupportedOperationException("Value " + value.getClass().getName() + " is unsupported."); } }
public static Writable toWritable(Object object) { if (object == null) { return null; //return NullWritable.get(); } if (object instanceof Writable) { return (Writable) object; } if (object instanceof String) { return new Text((String) object); } if (object instanceof Long) { return new VLongWritable((Long) object); } if (object instanceof Integer) { return new VIntWritable((Integer) object); } if (object instanceof Byte) { return new ByteWritable((Byte) object); } if (object instanceof Double) { return new DoubleWritable((Double) object); } if (object instanceof Float) { return new FloatWritable((Float) object); } if (object instanceof Boolean) { return new BooleanWritable((Boolean) object); } if (object instanceof byte[]) { return new BytesWritable((byte[]) object); } return new BytesWritable(object.toString().getBytes()); }
@Override protected void reduce(Text key, Iterable<VIntWritable> values, Context context) throws IOException, InterruptedException { int uriCount=0; long viewCount=0; for(VIntWritable views:values) { uriCount++; viewCount+=views.get(); } Text outKey=new Text(yrmo+" "+key.toString()); Text outValue=new Text(uriCount+" "+viewCount); context.write(outKey,outValue); }
@Override protected void map(Text key, Text value, Context context) throws IOException, InterruptedException { String keyString=key.toString(); try { Iterator<String> parts=SPACE_SPLITTER.split(keyString).iterator(); String siteId=parts.next(); String uri=parts.next(); int count=Integer.parseInt(value.toString()); context.write(new Text(siteId),new VIntWritable(count)); } catch(NoSuchElementException | NumberFormatException ex) { LOG.warn("Couldn't parse line ["+keyString+"]"); } }
@Test public void tryOne() throws IOException, InterruptedException { Text inKey=new Text("zr PallaPalla"); Text inValue=new Text("75312"); mapper.map(inKey,inValue,context); verify(context).write(new Text("zr"),new VIntWritable(75312)); }
@Test public void testPageId() throws IOException, InterruptedException { mapper.currentTag=new VIntWritable(1); Text line=new Text("<http://dbpedia.org/resource/AccessibleComputing> <http://dbpedia.org/ontology/wikiPageID> \"10\"^^<http://www.w3.org/2001/XMLSchema#integer>"); mapper.map(new LongWritable(101),line,context); verify(context).write( new TaggedTextItem("<http://dbpedia.org/resource/AccessibleComputing>",1), new TaggedTextItem("",1) ); }
@Test public void testRedirect() throws IOException, InterruptedException { mapper.currentTag=new VIntWritable(2); Text line=new Text("<http://dbpedia.org/resource/AsWeMayThink> <http://dbpedia.org/ontology/wikiPageRedirects> <http://dbpedia.org/resource/As_We_May_Think> .\n"); mapper.map(new LongWritable(101),line,context); verify(context).write( new TaggedTextItem("<http://dbpedia.org/resource/AsWeMayThink>",2), new TaggedTextItem("<http://dbpedia.org/resource/As_We_May_Think>",2) ); }