@Test public void testReadURL() throws Exception { HttpURLConnection conn = mock(HttpURLConnection.class); doReturn(new ByteArrayInputStream(FAKE_LOG_DATA)).when(conn).getInputStream(); doReturn(HttpURLConnection.HTTP_OK).when(conn).getResponseCode(); doReturn(Integer.toString(FAKE_LOG_DATA.length)).when(conn).getHeaderField("Content-Length"); URLConnectionFactory factory = mock(URLConnectionFactory.class); doReturn(conn).when(factory).openConnection(Mockito.<URL> any(), anyBoolean()); URL url = new URL("http://localhost/fakeLog"); EditLogInputStream elis = EditLogFileInputStream.fromUrl(factory, url, HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID, false); // Read the edit log and verify that we got all of the data. EnumMap<FSEditLogOpCodes, Holder<Integer>> counts = FSImageTestUtil .countEditLogOpTypes(elis); assertThat(counts.get(FSEditLogOpCodes.OP_ADD).held, is(1)); assertThat(counts.get(FSEditLogOpCodes.OP_SET_GENSTAMP_V1).held, is(1)); assertThat(counts.get(FSEditLogOpCodes.OP_CLOSE).held, is(1)); // Check that length header was picked up. assertEquals(FAKE_LOG_DATA.length, elis.length()); elis.close(); }
/** * @see #countEditLogOpTypes(File) */ public static EnumMap<FSEditLogOpCodes, Holder<Integer>> countEditLogOpTypes( EditLogInputStream elis) throws IOException { EnumMap<FSEditLogOpCodes, Holder<Integer>> opCounts = new EnumMap<FSEditLogOpCodes, Holder<Integer>>(FSEditLogOpCodes.class); FSEditLogOp op; while ((op = elis.readOp()) != null) { Holder<Integer> i = opCounts.get(op.opCode); if (i == null) { i = new Holder<Integer>(0); opCounts.put(op.opCode, i); } i.held++; } return opCounts; }
private long writeSegmentUntilCrash(MiniJournalCluster cluster, QuorumJournalManager qjm, long txid, int numTxns, Holder<Throwable> thrown) { long firstTxId = txid; long lastAcked = txid - 1; try { EditLogOutputStream stm = qjm.startLogSegment(txid, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); for (int i = 0; i < numTxns; i++) { QJMTestUtil.writeTxns(stm, txid++, 1); lastAcked++; } stm.close(); qjm.finalizeLogSegment(firstTxId, lastAcked); } catch (Throwable t) { thrown.held = t; } return lastAcked; }
@Test public void testReadURL() throws Exception { HttpURLConnection conn = mock(HttpURLConnection.class); doReturn(new ByteArrayInputStream(FAKE_LOG_DATA)).when(conn).getInputStream(); doReturn(HttpURLConnection.HTTP_OK).when(conn).getResponseCode(); doReturn(Integer.toString(FAKE_LOG_DATA.length)).when(conn).getHeaderField("Content-Length"); URLConnectionFactory factory = mock(URLConnectionFactory.class); doReturn(conn).when(factory).openConnection(Mockito.<URL> any(), anyBoolean()); URL url = new URL("http://localhost/fakeLog"); EditLogInputStream elis = EditLogFileInputStream.fromUrl(factory, url, HdfsServerConstants.INVALID_TXID, HdfsServerConstants.INVALID_TXID, false); // Read the edit log and verify that we got all of the data. EnumMap<FSEditLogOpCodes, Holder<Integer>> counts = FSImageTestUtil .countEditLogOpTypes(elis); assertThat(counts.get(FSEditLogOpCodes.OP_ADD).held, is(1)); assertThat(counts.get(FSEditLogOpCodes.OP_SET_GENSTAMP_V1).held, is(1)); assertThat(counts.get(FSEditLogOpCodes.OP_CLOSE).held, is(1)); // Check that length header was picked up. assertEquals(FAKE_LOG_DATA.length, elis.length()); elis.close(); }
private long writeSegmentUntilCrash(MiniJournalCluster cluster, QuorumJournalManager qjm, long txid, int numTxns, Holder<Throwable> thrown) { long firstTxId = txid; long lastAcked = txid - 1; try { EditLogOutputStream stm = qjm.startLogSegment(txid); for (int i = 0; i < numTxns; i++) { QJMTestUtil.writeTxns(stm, txid++, 1); lastAcked++; } stm.close(); qjm.finalizeLogSegment(firstTxId, lastAcked); } catch (Throwable t) { thrown.held = t; } return lastAcked; }
@Test public void testReadURL() throws Exception { // Start a simple web server which hosts the log data. HttpServer server = new HttpServer("test", "0.0.0.0", 0, true); server.start(); try { server.addServlet("fakeLog", "/fakeLog", FakeLogServlet.class); URL url = new URL("http://localhost:" + server.getPort() + "/fakeLog"); EditLogInputStream elis = EditLogFileInputStream.fromUrl( url, HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID, false); // Read the edit log and verify that we got all of the data. EnumMap<FSEditLogOpCodes, Holder<Integer>> counts = FSImageTestUtil.countEditLogOpTypes(elis); assertThat(counts.get(FSEditLogOpCodes.OP_ADD).held, is(1)); assertThat(counts.get(FSEditLogOpCodes.OP_SET_GENSTAMP_V1).held, is(1)); assertThat(counts.get(FSEditLogOpCodes.OP_CLOSE).held, is(1)); // Check that length header was picked up. assertEquals(FAKE_LOG_DATA.length, elis.length()); elis.close(); } finally { server.stop(); } }
private static void dumpOpCounts( EnumMap<FSEditLogOpCodes, Holder<Integer>> opCounts) { StringBuilder sb = new StringBuilder(); sb.append("Summary of operations loaded from edit log:\n "); Joiner.on("\n ").withKeyValueSeparator("=").appendTo(sb, opCounts); FSImage.LOG.debug(sb.toString()); }
private void incrOpCount(FSEditLogOpCodes opCode, EnumMap<FSEditLogOpCodes, Holder<Integer>> opCounts, Step step, Counter counter) { Holder<Integer> holder = opCounts.get(opCode); if (holder == null) { holder = new Holder<Integer>(1); opCounts.put(opCode, holder); } else { holder.held++; } counter.increment(); }
/** * @param editLog a path of an edit log file * @return the count of each type of operation in the log file * @throws Exception if there is an error reading it */ public static EnumMap<FSEditLogOpCodes,Holder<Integer>> countEditLogOpTypes( File editLog) throws Exception { EditLogInputStream elis = new EditLogFileInputStream(editLog); try { return countEditLogOpTypes(elis); } finally { IOUtils.closeStream(elis); } }
