static int run(DistributedFileSystem dfs, String[] argv, int idx) throws IOException { final RollingUpgradeAction action = RollingUpgradeAction.fromString( argv.length >= 2? argv[1]: ""); if (action == null) { throw new IllegalArgumentException("Failed to covert \"" + argv[1] +"\" to " + RollingUpgradeAction.class.getSimpleName()); } System.out.println(action + " rolling upgrade ..."); final RollingUpgradeInfo info = dfs.rollingUpgrade(action); switch(action){ case QUERY: break; case PREPARE: Preconditions.checkState(info.isStarted()); break; case FINALIZE: Preconditions.checkState(info == null || info.isFinalized()); break; } printMessage(info, System.out); return 0; }
private static void startRollingUpgrade(Path foo, Path bar, Path file, byte[] data, MiniDFSCluster cluster) throws IOException { final DistributedFileSystem dfs = cluster.getFileSystem(); //start rolling upgrade dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); dfs.rollingUpgrade(RollingUpgradeAction.PREPARE); dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); dfs.mkdirs(bar); Assert.assertTrue(dfs.exists(foo)); Assert.assertTrue(dfs.exists(bar)); //truncate a file final int newLength = DFSUtil.getRandom().nextInt(data.length - 1) + 1; dfs.truncate(file, newLength); TestFileTruncate.checkBlockRecovery(file, dfs); AppendTestUtil.checkFullFile(dfs, file, newLength, data); }
@Test (timeout = 300000) public void testQueryAfterRestart() throws IOException, InterruptedException { final Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); cluster.waitActive(); DistributedFileSystem dfs = cluster.getFileSystem(); dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); // start rolling upgrade dfs.rollingUpgrade(RollingUpgradeAction.PREPARE); queryForPreparation(dfs); dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); dfs.saveNamespace(); dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); cluster.restartNameNodes(); dfs.rollingUpgrade(RollingUpgradeAction.QUERY); } finally { if (cluster != null) { cluster.shutdown(); } } }
static void queryForPreparation(DistributedFileSystem dfs) throws IOException, InterruptedException { RollingUpgradeInfo info; int retries = 0; while (++retries < 10) { info = dfs.rollingUpgrade(RollingUpgradeAction.QUERY); if (info.createdRollbackImages()) { break; } Thread.sleep(1000); } if (retries >= 10) { Assert.fail("Query return false"); } }
@Override public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) throws IOException { final RollingUpgradeRequestProto r = RollingUpgradeRequestProto.newBuilder() .setAction(PBHelperClient.convert(action)).build(); try { final RollingUpgradeResponseProto proto = rpcProxy.rollingUpgrade(null, r); if (proto.hasRollingUpgradeInfo()) { return PBHelperClient.convert(proto.getRollingUpgradeInfo()); } return null; } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } }
private static void startRollingUpgrade(Path foo, Path bar, Path file, byte[] data, MiniDFSCluster cluster) throws IOException { final DistributedFileSystem dfs = cluster.getFileSystem(); //start rolling upgrade dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); dfs.rollingUpgrade(RollingUpgradeAction.PREPARE); dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); dfs.mkdirs(bar); Assert.assertTrue(dfs.exists(foo)); Assert.assertTrue(dfs.exists(bar)); //truncate a file final int newLength = ThreadLocalRandom.current().nextInt(data.length - 1) + 1; dfs.truncate(file, newLength); TestFileTruncate.checkBlockRecovery(file, dfs); AppendTestUtil.checkFullFile(dfs, file, newLength, data); }
static int run(DistributedFileSystem dfs, String[] argv, int idx) throws IOException { final RollingUpgradeAction action = RollingUpgradeAction.fromString( argv.length >= 2? argv[1]: ""); if (action == null) { throw new IllegalArgumentException("Failed to covert \"" + argv[1] +"\" to " + RollingUpgradeAction.class.getSimpleName()); } System.out.println(action + " rolling upgrade ..."); final RollingUpgradeInfo info = dfs.rollingUpgrade(action); switch(action){ case QUERY: break; case PREPARE: Preconditions.checkState(info.isStarted()); break; case FINALIZE: Preconditions.checkState(info.isFinalized()); break; } printMessage(info, System.out); return 0; }
RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) throws IOException { TraceScope scope = Trace.startSpan("rollingUpgrade", traceSampler); try { return namenode.rollingUpgrade(action); } finally { scope.close(); } }
@Override // ClientProtocol public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) throws IOException { checkNNStartup(); LOG.info("rollingUpgrade " + action); switch(action) { case QUERY: return namesystem.queryRollingUpgrade(); case PREPARE: return namesystem.startRollingUpgrade(); case FINALIZE: return namesystem.finalizeRollingUpgrade(); default: throw new UnsupportedActionException(action + " is not yet supported."); } }
public static RollingUpgradeActionProto convert(RollingUpgradeAction a) { switch (a) { case QUERY: return RollingUpgradeActionProto.QUERY; case PREPARE: return RollingUpgradeActionProto.START; case FINALIZE: return RollingUpgradeActionProto.FINALIZE; default: throw new IllegalArgumentException("Unexpected value: " + a); } }
public static RollingUpgradeAction convert(RollingUpgradeActionProto a) { switch (a) { case QUERY: return RollingUpgradeAction.QUERY; case START: return RollingUpgradeAction.PREPARE; case FINALIZE: return RollingUpgradeAction.FINALIZE; default: throw new IllegalArgumentException("Unexpected value: " + a); } }
@Override public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) throws IOException { final RollingUpgradeRequestProto r = RollingUpgradeRequestProto.newBuilder() .setAction(PBHelper.convert(action)).build(); try { final RollingUpgradeResponseProto proto = rpcProxy.rollingUpgrade(null, r); if (proto.hasRollingUpgradeInfo()) { return PBHelper.convert(proto.getRollingUpgradeInfo()); } return null; } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } }
@Test (timeout = 300000) public void testQuery() throws Exception { final Configuration conf = new Configuration(); MiniQJMHACluster cluster = null; try { cluster = new MiniQJMHACluster.Builder(conf).build(); MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); dfsCluster.transitionToActive(0); DistributedFileSystem dfs = dfsCluster.getFileSystem(0); dfsCluster.shutdownNameNode(1); // start rolling upgrade RollingUpgradeInfo info = dfs .rollingUpgrade(RollingUpgradeAction.PREPARE); Assert.assertTrue(info.isStarted()); info = dfs.rollingUpgrade(RollingUpgradeAction.QUERY); Assert.assertFalse(info.createdRollbackImages()); dfsCluster.restartNameNode(1); queryForPreparation(dfs); // The NN should have a copy of the fsimage in case of rollbacks. Assert.assertTrue(dfsCluster.getNamesystem(0).getFSImage() .hasRollbackFSImage()); } finally { if (cluster != null) { cluster.shutdown(); } } }
RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) throws IOException { checkOpen(); try (TraceScope ignored = tracer.newScope("rollingUpgrade")) { return namenode.rollingUpgrade(action); } }
private void testQuery(int nnCount) throws Exception{ final Configuration conf = new Configuration(); MiniQJMHACluster cluster = null; try { cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build(); MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); dfsCluster.transitionToActive(0); DistributedFileSystem dfs = dfsCluster.getFileSystem(0); // shutdown other NNs for (int i = 1; i < nnCount; i++) { dfsCluster.shutdownNameNode(i); } // start rolling upgrade RollingUpgradeInfo info = dfs .rollingUpgrade(RollingUpgradeAction.PREPARE); Assert.assertTrue(info.isStarted()); info = dfs.rollingUpgrade(RollingUpgradeAction.QUERY); Assert.assertFalse(info.createdRollbackImages()); // restart other NNs for (int i = 1; i < nnCount; i++) { dfsCluster.restartNameNode(i); } // check that one of the other NNs has created the rollback image and uploaded it queryForPreparation(dfs); // The NN should have a copy of the fsimage in case of rollbacks. Assert.assertTrue(dfsCluster.getNamesystem(0).getFSImage() .hasRollbackFSImage()); } finally { if (cluster != null) { cluster.shutdown(); } } }
public void testCheckpoint(int nnCount) throws IOException, InterruptedException { final Configuration conf = new Configuration(); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1); MiniQJMHACluster cluster = null; final Path foo = new Path("/foo"); try { cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build(); MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); dfsCluster.transitionToActive(0); DistributedFileSystem dfs = dfsCluster.getFileSystem(0); // start rolling upgrade RollingUpgradeInfo info = dfs .rollingUpgrade(RollingUpgradeAction.PREPARE); Assert.assertTrue(info.isStarted()); queryForPreparation(dfs); dfs.mkdirs(foo); long txid = dfs.rollEdits(); Assert.assertTrue(txid > 0); for(int i=1; i< nnCount; i++) { verifyNNCheckpoint(dfsCluster, txid, i); } } finally { if (cluster != null) { cluster.shutdown(); } } }