private void createCheckpoint(Path trashRoot, Date date) throws IOException { if (!fs.exists(new Path(trashRoot, CURRENT))) { return; } Path checkpointBase; synchronized (CHECKPOINT) { checkpointBase = new Path(trashRoot, CHECKPOINT.format(date)); } Path checkpoint = checkpointBase; Path current = new Path(trashRoot, CURRENT); int attempt = 0; while (true) { try { fs.rename(current, checkpoint, Rename.NONE); LOG.info("Created trash checkpoint: " + checkpoint.toUri().getPath()); break; } catch (FileAlreadyExistsException e) { if (++attempt > 1000) { throw new IOException("Failed to checkpoint trash: " + checkpoint); } checkpoint = checkpointBase.suffix("-" + attempt); } } }
@Test public void testRenameFileAsExistingFile() throws Exception { if (!renameSupported()) return; Path src = getTestRootPath(fc, "test/hadoop/file"); createFile(src); Path dst = getTestRootPath(fc, "test/new/existingFile"); createFile(dst); // Fails without overwrite option try { rename(src, dst, false, true, false, Rename.NONE); Assert.fail("Expected exception was not thrown"); } catch (IOException e) { Assert.assertTrue(unwrapException(e) instanceof FileAlreadyExistsException); } // Succeeds with overwrite option rename(src, dst, true, false, true, Rename.OVERWRITE); }
private void testRenameDirectoryAsNonExistentDirectory(Rename... options) throws Exception { if (!renameSupported()) return; Path src = getTestRootPath(fc, "test/hadoop/dir"); fc.mkdir(src, FileContext.DEFAULT_PERM, true); createFile(getTestRootPath(fc, "test/hadoop/dir/file1")); createFile(getTestRootPath(fc, "test/hadoop/dir/subdir/file2")); Path dst = getTestRootPath(fc, "test/new/newdir"); fc.mkdir(dst.getParent(), FileContext.DEFAULT_PERM, true); rename(src, dst, true, false, true, options); Assert.assertFalse("Nested file1 exists", exists(fc, getTestRootPath(fc, "test/hadoop/dir/file1"))); Assert.assertFalse("Nested file2 exists", exists(fc, getTestRootPath(fc, "test/hadoop/dir/subdir/file2"))); Assert.assertTrue("Renamed nested file1 exists", exists(fc, getTestRootPath(fc, "test/new/newdir/file1"))); Assert.assertTrue("Renamed nested exists", exists(fc, getTestRootPath(fc, "test/new/newdir/subdir/file2"))); }
@Test public void testRenameDirectoryAsEmptyDirectory() throws Exception { if (!renameSupported()) return; Path src = getTestRootPath(fc, "test/hadoop/dir"); fc.mkdir(src, FileContext.DEFAULT_PERM, true); createFile(getTestRootPath(fc, "test/hadoop/dir/file1")); createFile(getTestRootPath(fc, "test/hadoop/dir/subdir/file2")); Path dst = getTestRootPath(fc, "test/new/newdir"); fc.mkdir(dst, FileContext.DEFAULT_PERM, true); // Fails without overwrite option try { rename(src, dst, false, true, false, Rename.NONE); Assert.fail("Expected exception was not thrown"); } catch (IOException e) { // Expected (cannot over-write non-empty destination) Assert.assertTrue(unwrapException(e) instanceof FileAlreadyExistsException); } // Succeeds with the overwrite option rename(src, dst, true, false, true, Rename.OVERWRITE); }
@Test public void testRenameDirectoryAsFile() throws Exception { if (!renameSupported()) return; Path src = getTestRootPath(fc, "test/hadoop/dir"); fc.mkdir(src, FileContext.DEFAULT_PERM, true); Path dst = getTestRootPath(fc, "test/new/newfile"); createFile(dst); // Fails without overwrite option try { rename(src, dst, false, true, true, Rename.NONE); Assert.fail("Expected exception was not thrown"); } catch (IOException e) { } // Directory cannot be renamed as existing file try { rename(src, dst, false, true, true, Rename.OVERWRITE); Assert.fail("Expected exception was not thrown"); } catch (IOException ex) { } }
@Test(timeout=10000) /** Test creating a symlink using relative paths */ public void testCreateLinkUsingRelPaths() throws IOException { Path fileAbs = new Path(testBaseDir1(), "file"); Path linkAbs = new Path(testBaseDir1(), "linkToFile"); Path schemeAuth = new Path(testURI().toString()); Path fileQual = new Path(schemeAuth, testBaseDir1()+"/file"); createAndWriteFile(fileAbs); wrapper.setWorkingDirectory(new Path(testBaseDir1())); wrapper.createSymlink(new Path("file"), new Path("linkToFile"), false); checkLink(linkAbs, new Path("file"), fileQual); // Now rename the link's parent. Because the target was specified // with a relative path the link should still resolve. Path dir1 = new Path(testBaseDir1()); Path dir2 = new Path(testBaseDir2()); Path linkViaDir2 = new Path(testBaseDir2(), "linkToFile"); Path fileViaDir2 = new Path(schemeAuth, testBaseDir2()+"/file"); wrapper.rename(dir1, dir2, Rename.OVERWRITE); FileStatus[] stats = wrapper.listStatus(dir2); assertEquals(fileViaDir2, wrapper.getFileLinkStatus(linkViaDir2).getSymlink()); readFile(linkViaDir2); }
@Test(timeout=10000) /** Test creating a symlink using absolute paths */ public void testCreateLinkUsingAbsPaths() throws IOException { Path fileAbs = new Path(testBaseDir1()+"/file"); Path linkAbs = new Path(testBaseDir1()+"/linkToFile"); Path schemeAuth = new Path(testURI().toString()); Path fileQual = new Path(schemeAuth, testBaseDir1()+"/file"); createAndWriteFile(fileAbs); wrapper.createSymlink(fileAbs, linkAbs, false); checkLink(linkAbs, fileAbs, fileQual); // Now rename the link's parent. The target doesn't change and // now no longer exists so accessing the link should fail. Path dir1 = new Path(testBaseDir1()); Path dir2 = new Path(testBaseDir2()); Path linkViaDir2 = new Path(testBaseDir2(), "linkToFile"); wrapper.rename(dir1, dir2, Rename.OVERWRITE); assertEquals(fileQual, wrapper.getFileLinkStatus(linkViaDir2).getSymlink()); try { readFile(linkViaDir2); fail("The target should not exist"); } catch (FileNotFoundException x) { // Expected } }
@Test(timeout=10000) /** Test rename a directory to a symlink to a directory */ public void testRenameDirToSymlinkToDir() throws IOException { Path dir1 = new Path(testBaseDir1()); Path subDir = new Path(testBaseDir2(), "subDir"); Path linkToDir = new Path(testBaseDir2(), "linkToDir"); wrapper.mkdir(subDir, FileContext.DEFAULT_PERM, false); wrapper.createSymlink(subDir, linkToDir, false); try { wrapper.rename(dir1, linkToDir, Rename.OVERWRITE); fail("Renamed directory to a symlink"); } catch (IOException e) { // Expected. Both must be directories. assertTrue(unwrapException(e) instanceof IOException); } assertTrue(wrapper.exists(dir1)); assertTrue(wrapper.exists(linkToDir)); }
@Test(timeout=10000) /** Test rename a directory to a symlink to a file */ public void testRenameDirToSymlinkToFile() throws IOException { Path dir1 = new Path(testBaseDir1()); Path file = new Path(testBaseDir2(), "file"); Path linkToFile = new Path(testBaseDir2(), "linkToFile"); createAndWriteFile(file); wrapper.createSymlink(file, linkToFile, false); try { wrapper.rename(dir1, linkToFile, Rename.OVERWRITE); fail("Renamed directory to a symlink"); } catch (IOException e) { // Expected. Both must be directories. assertTrue(unwrapException(e) instanceof IOException); } assertTrue(wrapper.exists(dir1)); assertTrue(wrapper.exists(linkToFile)); }
@Test(timeout=10000) /** Test rename a directory to a dangling symlink */ public void testRenameDirToDanglingSymlink() throws IOException { Path dir = new Path(testBaseDir1()); Path link = new Path(testBaseDir2(), "linkToFile"); wrapper.createSymlink(new Path("/doesNotExist"), link, false); try { wrapper.rename(dir, link, Rename.OVERWRITE); fail("Renamed directory to a symlink"); } catch (IOException e) { // Expected. Both must be directories. assertTrue(unwrapException(e) instanceof IOException); } assertTrue(wrapper.exists(dir)); assertTrue(wrapper.getFileLinkStatus(link) != null); }
@Test(timeout=10000) /** Test rename a file to a symlink to a directory */ public void testRenameFileToSymlinkToDir() throws IOException { Path file = new Path(testBaseDir1(), "file"); Path subDir = new Path(testBaseDir1(), "subDir"); Path link = new Path(testBaseDir1(), "link"); wrapper.mkdir(subDir, FileContext.DEFAULT_PERM, false); wrapper.createSymlink(subDir, link, false); createAndWriteFile(file); try { wrapper.rename(file, link); fail("Renamed file to symlink w/o overwrite"); } catch (IOException e) { // Expected assertTrue(unwrapException(e) instanceof FileAlreadyExistsException); } wrapper.rename(file, link, Rename.OVERWRITE); assertFalse(wrapper.exists(file)); assertTrue(wrapper.exists(link)); assertTrue(wrapper.isFile(link)); assertFalse(wrapper.getFileLinkStatus(link).isSymlink()); }
@Test(timeout=10000) /** Test rename a file to a symlink to a file */ public void testRenameFileToSymlinkToFile() throws IOException { Path file1 = new Path(testBaseDir1(), "file1"); Path file2 = new Path(testBaseDir1(), "file2"); Path link = new Path(testBaseDir1(), "linkToFile"); createAndWriteFile(file1); createAndWriteFile(file2); wrapper.createSymlink(file2, link, false); try { wrapper.rename(file1, link); fail("Renamed file to symlink w/o overwrite"); } catch (IOException e) { // Expected assertTrue(unwrapException(e) instanceof FileAlreadyExistsException); } wrapper.rename(file1, link, Rename.OVERWRITE); assertFalse(wrapper.exists(file1)); assertTrue(wrapper.exists(link)); assertTrue(wrapper.isFile(link)); assertFalse(wrapper.getFileLinkStatus(link).isSymlink()); }
@Test(timeout=10000) /** Test rename a file to a dangling symlink */ public void testRenameFileToDanglingSymlink() throws IOException { /* NB: Local file system doesn't handle dangling links correctly * since File.exists(danglinLink) returns false. */ if ("file".equals(getScheme())) { return; } Path file1 = new Path(testBaseDir1(), "file1"); Path link = new Path(testBaseDir1(), "linkToFile"); createAndWriteFile(file1); wrapper.createSymlink(new Path("/doesNotExist"), link, false); try { wrapper.rename(file1, link); } catch (IOException e) { // Expected } wrapper.rename(file1, link, Rename.OVERWRITE); assertFalse(wrapper.exists(file1)); assertTrue(wrapper.exists(link)); assertTrue(wrapper.isFile(link)); assertFalse(wrapper.getFileLinkStatus(link).isSymlink()); }
@Test(timeout=10000) /** Rename a symlink to a file that exists */ public void testRenameSymlinkToExistingFile() throws IOException { Path file1 = new Path(testBaseDir1(), "file"); Path file2 = new Path(testBaseDir1(), "someFile"); Path link = new Path(testBaseDir1(), "linkToFile"); createAndWriteFile(file1); createAndWriteFile(file2); wrapper.createSymlink(file2, link, false); try { wrapper.rename(link, file1); fail("Renamed w/o passing overwrite"); } catch (IOException e) { // Expected assertTrue(unwrapException(e) instanceof FileAlreadyExistsException); } wrapper.rename(link, file1, Rename.OVERWRITE); assertFalse(wrapper.exists(link)); assertTrue(wrapper.getFileLinkStatus(file1).isSymlink()); assertEquals(file2, wrapper.getLinkTarget(file1)); }
@Test(timeout=10000) /** Test rename the symlink's target */ public void testRenameLinkTarget() throws IOException { Path file = new Path(testBaseDir1(), "file"); Path fileNew = new Path(testBaseDir1(), "fileNew"); Path link = new Path(testBaseDir1(), "linkToFile"); createAndWriteFile(file); wrapper.createSymlink(file, link, false); wrapper.rename(file, fileNew, Rename.OVERWRITE); try { readFile(link); fail("Link should be dangling"); } catch (IOException x) { // Expected } wrapper.rename(fileNew, file, Rename.OVERWRITE); readFile(link); }
@Test public void testRenameFileAsExistingFile() throws Exception { if (!renameSupported()) return; Path src = getTestRootPath(fSys, "test/hadoop/file"); createFile(src); Path dst = getTestRootPath(fSys, "test/new/existingFile"); createFile(dst); // Fails without overwrite option try { rename(src, dst, false, true, false, Rename.NONE); Assert.fail("Expected exception was not thrown"); } catch (IOException e) { Assert.assertTrue(unwrapException(e) instanceof FileAlreadyExistsException); } // Succeeds with overwrite option rename(src, dst, true, false, true, Rename.OVERWRITE); }
private void doTestRenameDirectoryAsNonExistentDirectory(Rename... options) throws Exception { if (!renameSupported()) return; Path src = getTestRootPath(fSys, "test/hadoop/dir"); fSys.mkdirs(src); createFile(getTestRootPath(fSys, "test/hadoop/dir/file1")); createFile(getTestRootPath(fSys, "test/hadoop/dir/subdir/file2")); Path dst = getTestRootPath(fSys, "test/new/newdir"); fSys.mkdirs(dst.getParent()); rename(src, dst, true, false, true, options); Assert.assertFalse("Nested file1 exists", exists(fSys, getTestRootPath(fSys, "test/hadoop/dir/file1"))); Assert.assertFalse("Nested file2 exists", exists(fSys, getTestRootPath(fSys, "test/hadoop/dir/subdir/file2"))); Assert.assertTrue("Renamed nested file1 exists", exists(fSys, getTestRootPath(fSys, "test/new/newdir/file1"))); Assert.assertTrue("Renamed nested exists", exists(fSys, getTestRootPath(fSys, "test/new/newdir/subdir/file2"))); }
@Test public void testRenameDirectoryAsEmptyDirectory() throws Exception { if (!renameSupported()) return; Path src = getTestRootPath(fSys, "test/hadoop/dir"); fSys.mkdirs(src); createFile(getTestRootPath(fSys, "test/hadoop/dir/file1")); createFile(getTestRootPath(fSys, "test/hadoop/dir/subdir/file2")); Path dst = getTestRootPath(fSys, "test/new/newdir"); fSys.mkdirs(dst); // Fails without overwrite option try { rename(src, dst, false, true, false, Rename.NONE); Assert.fail("Expected exception was not thrown"); } catch (IOException e) { // Expected (cannot over-write non-empty destination) Assert.assertTrue(unwrapException(e) instanceof FileAlreadyExistsException); } // Succeeds with the overwrite option rename(src, dst, true, false, true, Rename.OVERWRITE); }
@Test public void testRenameDirectoryAsFile() throws Exception { if (!renameSupported()) return; Path src = getTestRootPath(fSys, "test/hadoop/dir"); fSys.mkdirs(src); Path dst = getTestRootPath(fSys, "test/new/newfile"); createFile(dst); // Fails without overwrite option try { rename(src, dst, false, true, true, Rename.NONE); Assert.fail("Expected exception was not thrown"); } catch (IOException e) { } // Directory cannot be renamed as existing file try { rename(src, dst, false, true, true, Rename.OVERWRITE); Assert.fail("Expected exception was not thrown"); } catch (IOException ex) { } }
@Override protected void toXml(ContentHandler contentHandler) throws SAXException { XMLUtils.addSaxString(contentHandler, "LENGTH", Integer.toString(length)); XMLUtils.addSaxString(contentHandler, "SRC", src); XMLUtils.addSaxString(contentHandler, "DST", dst); XMLUtils.addSaxString(contentHandler, "TIMESTAMP", Long.toString(timestamp)); StringBuilder bld = new StringBuilder(); String prefix = ""; for (Rename r : options) { bld.append(prefix).append(r.toString()); prefix = "|"; } XMLUtils.addSaxString(contentHandler, "OPTIONS", bld.toString()); appendRpcIdsToXml(contentHandler, rpcClientId, rpcCallId); }
@Override void fromXml(Stanza st) throws InvalidXmlException { this.length = Integer.parseInt(st.getValue("LENGTH")); this.src = st.getValue("SRC"); this.dst = st.getValue("DST"); this.timestamp = Long.parseLong(st.getValue("TIMESTAMP")); String opts = st.getValue("OPTIONS"); String o[] = opts.split("\\|"); this.options = new Rename[o.length]; for (int i = 0; i < o.length; i++) { if (o[i].equals("")) continue; try { this.options[i] = Rename.valueOf(o[i]); } finally { if (this.options[i] == null) { System.err.println("error parsing Rename value: \"" + o[i] + "\""); } } } readRpcIdsFromXml(st); }
@Override public void rename2(String src, String dst, Rename... options) throws AccessControlException, DSQuotaExceededException, FileAlreadyExistsException, FileNotFoundException, NSQuotaExceededException, ParentNotDirectoryException, SafeModeException, UnresolvedLinkException, IOException { boolean overwrite = false; if (options != null) { for (Rename option : options) { if (option == Rename.OVERWRITE) { overwrite = true; } } } Rename2RequestProto req = Rename2RequestProto.newBuilder(). setSrc(src). setDst(dst).setOverwriteDest(overwrite). build(); try { rpcProxy.rename2(null, req); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } }
private void doRenameLinkTargetNotWritableFC() throws Exception { // Rename the link when the target and parent are not writable user.doAs(new PrivilegedExceptionAction<Object>() { @Override public Object run() throws IOException { // First FileContext FileContext myfc = FileContext.getFileContext(conf); Path newlink = new Path(linkParent, "newlink"); myfc.rename(link, newlink, Rename.NONE); Path linkTarget = myfc.getLinkTarget(newlink); assertEquals("Expected link's target to match target!", target, linkTarget); return null; } }); assertTrue("Expected target to exist", wrapper.exists(target)); }
private void doRenameSrcNotWritableFC() throws Exception { // Rename the link when the target and parent are not writable try { user.doAs(new PrivilegedExceptionAction<Object>() { @Override public Object run() throws IOException { FileContext myfc = FileContext.getFileContext(conf); Path newlink = new Path(targetParent, "newlink"); myfc.rename(link, newlink, Rename.NONE); return null; } }); fail("Renamed link even though link's parent is not writable!"); } catch (IOException e) { GenericTestUtils.assertExceptionContains("Permission denied", e); } }
/** * Perform operations such as setting quota, deletion of files, rename and * ensure system can apply edits log during startup. */ @Test public void testEditsLogRename() throws Exception { DistributedFileSystem fs = cluster.getFileSystem(); Path src1 = getTestRootPath(fc, "testEditsLogRename/srcdir/src1"); Path dst1 = getTestRootPath(fc, "testEditsLogRename/dstdir/dst1"); createFile(src1); fs.mkdirs(dst1.getParent()); createFile(dst1); // Set quota so that dst1 parent cannot allow under it new files/directories fs.setQuota(dst1.getParent(), 2, HdfsConstants.QUOTA_DONT_SET); // Free up quota for a subsequent rename fs.delete(dst1, true); rename(src1, dst1, true, true, false, Rename.OVERWRITE); // Restart the cluster and ensure the above operations can be // loaded from the edits log restartCluster(); fs = cluster.getFileSystem(); src1 = getTestRootPath(fc, "testEditsLogRename/srcdir/src1"); dst1 = getTestRootPath(fc, "testEditsLogRename/dstdir/dst1"); Assert.assertFalse(fs.exists(src1)); // ensure src1 is already renamed Assert.assertTrue(fs.exists(dst1)); // ensure rename dst exists }
/** * Test for rename2 */ @Test public void testRename2() throws Exception { String src = "/testNamenodeRetryCache/testRename2/src"; String target = "/testNamenodeRetryCache/testRename2/target"; resetCall(); nnRpc.mkdirs(src, perm, true); // Retried renames succeed newCall(); nnRpc.rename2(src, target, Rename.NONE); nnRpc.rename2(src, target, Rename.NONE); nnRpc.rename2(src, target, Rename.NONE); // A non-retried request fails newCall(); try { nnRpc.rename2(src, target, Rename.NONE); Assert.fail("testRename 2 expected exception is not thrown"); } catch (IOException e) { // expected } }
/** Rename test when src and dst exist */ @Test public void testFailuresExistingDst() throws Exception { final Path src = getTestPath("testFailuresExistingDst/dir/src"); final Path dst = getTestPath("testFailuresExistingDst/newdir/dst"); createFile(src); createFile(dst); // During rename, while removing src, an exception is thrown TestFiRename.exceptionOnRemove(src.toString()); rename(src, dst, true, true, true, Rename.OVERWRITE); // During rename, while removing dst, an exception is thrown TestFiRename.exceptionOnRemove(dst.toString()); rename(src, dst, true, true, true, Rename.OVERWRITE); // During rename, while adding dst an exception is thrown TestFiRename.exceptionOnAdd(dst.toString()); rename(src, dst, true, true, true, Rename.OVERWRITE); }
@Test(timeout=10000) /** Rename a symlink to a file that exists */ public void testRenameSymlinkToExistingFile() throws IOException { Path file1 = new Path(testBaseDir1(), "file"); Path file2 = new Path(testBaseDir1(), "someFile"); Path link = new Path(testBaseDir1(), "linkToFile"); createAndWriteFile(file1); createAndWriteFile(file2); wrapper.createSymlink(file2, link, false); try { wrapper.rename(link, file1); fail("Renamed w/o passing overwrite"); } catch (IOException e) { // Expected assertTrue(unwrapException(e) instanceof FileAlreadyExistsException); } wrapper.rename(link, file1, Rename.OVERWRITE); assertFalse(wrapper.exists(link)); if (!emulatingSymlinksOnWindows()) { assertTrue(wrapper.getFileLinkStatus(file1).isSymlink()); assertEquals(file2, wrapper.getLinkTarget(file1)); } }