void testFileCrcInternal(boolean inlineChecksum) throws IOException { ((Log4JLogger) HftpFileSystem.LOG).getLogger().setLevel(Level.ALL); Random random = new Random(1); final long seed = random.nextLong(); random.setSeed(seed); FileSystem fs = FileSystem.getLocal(new Configuration()); // generate random data final byte[] data = new byte[1024 * 1024 + 512 * 7 + 66]; random.nextBytes(data); // write data to a file Path foo = new Path(TEST_ROOT_DIR, "foo_" + inlineChecksum); { final FSDataOutputStream out = fs.create(foo, false, 512, (short) 2, 512); out.write(data); out.close(); } // compute data CRC DataChecksum checksum = DataChecksum.newDataChecksum( DataChecksum.CHECKSUM_CRC32, 1); checksum.update(data, 0, data.length); // compute checksum final int crc = fs.getFileCrc(foo); System.out.println("crc=" + crc); TestCase.assertEquals((int) checksum.getValue(), crc); }
static public Credentials getDTfromRemote(String nnAddr, String renewer) throws IOException { DataInputStream dis = null; InetSocketAddress serviceAddr = NetUtils.createSocketAddr(nnAddr); try { StringBuffer url = new StringBuffer(); if (renewer != null) { url.append(nnAddr).append(GetDelegationTokenServlet.PATH_SPEC) .append("?").append(GetDelegationTokenServlet.RENEWER).append("=") .append(renewer); } else { url.append(nnAddr).append(GetDelegationTokenServlet.PATH_SPEC); } URL remoteURL = new URL(url.toString()); URLConnection connection = SecurityUtil.openSecureHttpConnection(remoteURL); InputStream in = connection.getInputStream(); Credentials ts = new Credentials(); dis = new DataInputStream(in); ts.readFields(dis); for(Token<?> token: ts.getAllTokens()) { token.setKind(HftpFileSystem.TOKEN_KIND); SecurityUtil.setTokenService(token, serviceAddr); } return ts; } catch (Exception e) { throw new IOException("Unable to obtain remote token", e); } finally { if(dis != null) dis.close(); } }
@Test public void testLazyRenewerStartup() { DelegationTokenRenewer<HftpFileSystem> dtr = new DelegationTokenRenewer<HftpFileSystem>(HftpFileSystem.class); assertFalse(dtr.isAlive()); dtr.start(); assertFalse(dtr.isAlive()); dtr.addRenewAction(null); assertTrue(dtr.isAlive()); }
protected SimpleDateFormat initialValue() { return HftpFileSystem.getDateFormat(); }
@Override protected SimpleDateFormat initialValue() { return HftpFileSystem.getDateFormat(); }
/** * Utility method to obtain a delegation token over http * @param nnAddr Namenode http addr, such as http://namenode:50070 * @param renewer User that is renewing the ticket in such a request */ static public Credentials getDTfromRemote(String nnAddr, String renewer) throws IOException { DataInputStream dis = null; InetSocketAddress serviceAddr = NetUtils.createSocketAddr(nnAddr); try { StringBuffer url = new StringBuffer(); if (renewer != null) { url.append(nnAddr).append(GetDelegationTokenServlet.PATH_SPEC).append("?"). append(GetDelegationTokenServlet.RENEWER).append("=").append(renewer); } else { url.append(nnAddr).append(GetDelegationTokenServlet.PATH_SPEC); } if(LOG.isDebugEnabled()) { LOG.debug("Retrieving token from: " + url); } URL remoteURL = new URL(url.toString()); boolean isSecure = "https".equals(remoteURL.getProtocol().toLowerCase()); URLConnection connection = SecurityUtil.openSecureHttpConnection(remoteURL); InputStream in = connection.getInputStream(); Credentials ts = new Credentials(); dis = new DataInputStream(in); ts.readFields(dis); for(Token<?> token: ts.getAllTokens()) { if (isSecure) { token.setKind(HsftpFileSystem.TOKEN_KIND); } else { token.setKind(HftpFileSystem.TOKEN_KIND); } SecurityUtil.setTokenService(token, serviceAddr); } return ts; } catch (Exception e) { throw new IOException("Unable to obtain remote token", e); } finally { if(dis != null) dis.close(); } }