/** Handle HTTP GET request for the root. */ @GET @Path("/") @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) public Response getRoot( @Context final UserGroupInformation ugi, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(NamenodeAddressParam.NAME) @DefaultValue(NamenodeAddressParam.DEFAULT) final NamenodeAddressParam namenode, @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) final GetOpParam op, @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) final OffsetParam offset, @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT) final LengthParam length, @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) final BufferSizeParam bufferSize ) throws IOException, InterruptedException { return get(ugi, delegation, namenode, ROOT, op, offset, length, bufferSize); }
/** Handle HTTP GET request for the root. */ @GET @Path("/") @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) public Response getRoot( @Context final UserGroupInformation ugi, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) final UserParam username, @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) final DoAsParam doAsUser, @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) final GetOpParam op, @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) final OffsetParam offset, @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT) final LengthParam length, @QueryParam(RenewerParam.NAME) @DefaultValue(RenewerParam.DEFAULT) final RenewerParam renewer, @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) final BufferSizeParam bufferSize ) throws IOException, InterruptedException { return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length, renewer, bufferSize); }
/** Handle HTTP GET request for the root. */ @GET @Path("/") @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) public Response getRoot( @Context final UserGroupInformation ugi, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(NamenodeRpcAddressParam.NAME) @DefaultValue(NamenodeRpcAddressParam.DEFAULT) final NamenodeRpcAddressParam namenodeRpcAddress, @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) final GetOpParam op, @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) final OffsetParam offset, @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT) final LengthParam length, @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) final BufferSizeParam bufferSize ) throws IOException, InterruptedException { return get(ugi, delegation, namenodeRpcAddress, ROOT, op, offset, length, bufferSize); }
/** Handle HTTP GET request for the root. */ @GET @Path("/") @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) public Response getRoot( @Context final UserGroupInformation ugi, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) final UserParam username, @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) final DoAsParam doAsUser, @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) final GetOpParam op, @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) final OffsetParam offset, @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT) final LengthParam length, @QueryParam(RenewerParam.NAME) @DefaultValue(RenewerParam.DEFAULT) final RenewerParam renewer, @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) final BufferSizeParam bufferSize ) throws IOException, URISyntaxException, InterruptedException { return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length, renewer, bufferSize); }
/** Handle HTTP GET request for the root. */ @GET @Path("/") @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) public Response getRoot( @Context final UserGroupInformation ugi, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) final GetOpParam op, @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) final OffsetParam offset, @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT) final LengthParam length, @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) final BufferSizeParam bufferSize ) throws IOException, InterruptedException { return get(ugi, delegation, ROOT, op, offset, length, bufferSize); }
@Test public void testWebHdfsOffsetAndLength() throws Exception{ MiniDFSCluster cluster = null; final Configuration conf = WebHdfsTestUtil.createConf(); final int OFFSET = 42; final int LENGTH = 512; final String PATH = "/foo"; byte[] CONTENTS = new byte[1024]; RANDOM.nextBytes(CONTENTS); try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); final WebHdfsFileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsFileSystem.SCHEME); try (OutputStream os = fs.create(new Path(PATH))) { os.write(CONTENTS); } InetSocketAddress addr = cluster.getNameNode().getHttpAddress(); URL url = new URL("http", addr.getHostString(), addr .getPort(), WebHdfsFileSystem.PATH_PREFIX + PATH + "?op=OPEN" + Param.toSortedString("&", new OffsetParam((long) OFFSET), new LengthParam((long) LENGTH)) ); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setInstanceFollowRedirects(true); Assert.assertEquals(LENGTH, conn.getContentLength()); byte[] subContents = new byte[LENGTH]; byte[] realContents = new byte[LENGTH]; System.arraycopy(CONTENTS, OFFSET, subContents, 0, LENGTH); IOUtils.readFully(conn.getInputStream(), realContents); Assert.assertArrayEquals(subContents, realContents); } finally { if (cluster != null) { cluster.shutdown(); } } }
@Test public void testWebHdfsOffsetAndLength() throws Exception{ MiniDFSCluster cluster = null; final Configuration conf = WebHdfsTestUtil.createConf(); final int OFFSET = 42; final int LENGTH = 512; final String PATH = "/foo"; byte[] CONTENTS = new byte[1024]; RANDOM.nextBytes(CONTENTS); try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); final WebHdfsFileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME); try (OutputStream os = fs.create(new Path(PATH))) { os.write(CONTENTS); } InetSocketAddress addr = cluster.getNameNode().getHttpAddress(); URL url = new URL("http", addr.getHostString(), addr .getPort(), WebHdfsFileSystem.PATH_PREFIX + PATH + "?op=OPEN" + Param.toSortedString("&", new OffsetParam((long) OFFSET), new LengthParam((long) LENGTH)) ); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setInstanceFollowRedirects(true); Assert.assertEquals(LENGTH, conn.getContentLength()); byte[] subContents = new byte[LENGTH]; byte[] realContents = new byte[LENGTH]; System.arraycopy(CONTENTS, OFFSET, subContents, 0, LENGTH); IOUtils.readFully(conn.getInputStream(), realContents); Assert.assertArrayEquals(subContents, realContents); } finally { if (cluster != null) { cluster.shutdown(); } } }
/** Handle HTTP GET request. */ @GET @Path("{" + UriFsPathParam.NAME + ":.*}") @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) public Response get( @Context final UserGroupInformation ugi, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(NamenodeAddressParam.NAME) @DefaultValue(NamenodeAddressParam.DEFAULT) final NamenodeAddressParam namenode, @PathParam(UriFsPathParam.NAME) final UriFsPathParam path, @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) final GetOpParam op, @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) final OffsetParam offset, @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT) final LengthParam length, @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) final BufferSizeParam bufferSize ) throws IOException, InterruptedException { final String nnId = namenode.getValue(); init(ugi, delegation, nnId, path, op, offset, length, bufferSize); return ugi.doAs(new PrivilegedExceptionAction<Response>() { @Override public Response run() throws IOException { return get(nnId, path.getAbsolutePath(), op, offset, length, bufferSize); } }); }
@Override public BlockLocation[] getFileBlockLocations(final Path p, final long offset, final long length) throws IOException { statistics.incrementReadOps(1); final HttpOpParam.Op op = GetOpParam.Op.GET_BLOCK_LOCATIONS; final Map<?, ?> m = run(op, p, new OffsetParam(offset), new LengthParam(length)); return DFSUtil.locatedBlocks2Locations(JsonUtil.toLocatedBlocks(m)); }
/** Handle HTTP GET request. */ @GET @Path("{" + UriFsPathParam.NAME + ":.*}") @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) public Response get( @Context final UserGroupInformation ugi, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(NamenodeRpcAddressParam.NAME) @DefaultValue(NamenodeRpcAddressParam.DEFAULT) final NamenodeRpcAddressParam namenodeRpcAddress, @PathParam(UriFsPathParam.NAME) final UriFsPathParam path, @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) final GetOpParam op, @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) final OffsetParam offset, @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT) final LengthParam length, @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) final BufferSizeParam bufferSize ) throws IOException, InterruptedException { final InetSocketAddress nnRpcAddr = namenodeRpcAddress.getValue(); init(ugi, delegation, nnRpcAddr, path, op, offset, length, bufferSize); return ugi.doAs(new PrivilegedExceptionAction<Response>() { @Override public Response run() throws IOException { return get(ugi, delegation, nnRpcAddr, path.getAbsolutePath(), op, offset, length, bufferSize); } }); }
/** * Handle HTTP GET request for the root. */ @GET @Path("/") @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) public Response getRoot( @Context final UserGroupInformation ugi, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) final UserParam username, @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) final DoAsParam doAsUser, @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) final GetOpParam op, @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) final OffsetParam offset, @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT) final LengthParam length, @QueryParam(RenewerParam.NAME) @DefaultValue(RenewerParam.DEFAULT) final RenewerParam renewer, @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) final BufferSizeParam bufferSize, @QueryParam(FsActionParam.NAME) @DefaultValue(FsActionParam.DEFAULT) final FsActionParam fsAction) throws IOException, InterruptedException { return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length, renewer, bufferSize, fsAction); }
/** * Handle HTTP GET request for the root. */ @GET @Path("/") @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) public Response getRoot( @Context final UserGroupInformation ugi, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(NamenodeRpcAddressParam.NAME) @DefaultValue(NamenodeRpcAddressParam.DEFAULT) final NamenodeRpcAddressParam namenodeRpcAddress, @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) final GetOpParam op, @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) final OffsetParam offset, @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT) final LengthParam length, @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) final BufferSizeParam bufferSize) throws IOException, InterruptedException { return get(ugi, delegation, namenodeRpcAddress, ROOT, op, offset, length, bufferSize); }
/** * Handle HTTP GET request. */ @GET @Path("{" + UriFsPathParam.NAME + ":.*}") @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) public Response get( @Context final UserGroupInformation ugi, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(NamenodeRpcAddressParam.NAME) @DefaultValue(NamenodeRpcAddressParam.DEFAULT) final NamenodeRpcAddressParam namenodeRpcAddress, @PathParam(UriFsPathParam.NAME) final UriFsPathParam path, @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) final GetOpParam op, @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) final OffsetParam offset, @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT) final LengthParam length, @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) final BufferSizeParam bufferSize) throws IOException, InterruptedException { final InetSocketAddress nnRpcAddr = namenodeRpcAddress.getValue(); init(ugi, delegation, nnRpcAddr, path, op, offset, length, bufferSize); return ugi.doAs(new PrivilegedExceptionAction<Response>() { @Override public Response run() throws IOException { return get(ugi, delegation, nnRpcAddr, path.getAbsolutePath(), op, offset, length, bufferSize); } }); }
@Override public BlockLocation[] getFileBlockLocations(final FileStatus status, final long offset, final long length) throws IOException { if (status == null) { return null; } statistics.incrementReadOps(1); final Path p = status.getPath(); final HttpOpParam.Op op = GetOpParam.Op.GET_BLOCK_LOCATIONS; final Map<?, ?> m = run(op, p, new OffsetParam(offset), new LengthParam(length)); return DFSUtil.locatedBlocks2Locations(JsonUtil.toLocatedBlocks(m)); }
long length() { return new LengthParam(param(LengthParam.NAME)).getLength(); }
@Test public void testUGICacheSecure() throws Exception { // fake turning on security so api thinks it should use tokens SecurityUtil.setAuthenticationMethod(KERBEROS, conf); UserGroupInformation.setConfiguration(conf); UserGroupInformation ugi = UserGroupInformation .createRemoteUser("test-user"); ugi.setAuthenticationMethod(KERBEROS); ugi = UserGroupInformation.createProxyUser("test-proxy-user", ugi); UserGroupInformation.setLoginUser(ugi); List<Token<DelegationTokenIdentifier>> tokens = Lists.newArrayList(); getWebHdfsFileSystem(ugi, conf, tokens); String uri1 = WebHdfsFileSystem.PATH_PREFIX + PATH + "?op=OPEN" + Param.toSortedString("&", new NamenodeAddressParam("127.0.0.1:1010"), new OffsetParam((long) OFFSET), new LengthParam((long) LENGTH), new DelegationParam(tokens.get(0).encodeToUrlString())); String uri2 = WebHdfsFileSystem.PATH_PREFIX + PATH + "?op=OPEN" + Param.toSortedString("&", new NamenodeAddressParam("127.0.0.1:1010"), new OffsetParam((long) OFFSET), new LengthParam((long) LENGTH), new DelegationParam(tokens.get(1).encodeToUrlString())); DataNodeUGIProvider ugiProvider1 = new DataNodeUGIProvider( new ParameterParser(new QueryStringDecoder(URI.create(uri1)), conf)); UserGroupInformation ugi11 = ugiProvider1.ugi(); UserGroupInformation ugi12 = ugiProvider1.ugi(); Assert.assertEquals( "With UGI cache, two UGIs returned by the same token should be same", ugi11, ugi12); DataNodeUGIProvider ugiProvider2 = new DataNodeUGIProvider( new ParameterParser(new QueryStringDecoder(URI.create(uri2)), conf)); UserGroupInformation url21 = ugiProvider2.ugi(); UserGroupInformation url22 = ugiProvider2.ugi(); Assert.assertEquals( "With UGI cache, two UGIs returned by the same token should be same", url21, url22); Assert.assertNotEquals( "With UGI cache, two UGIs for the different token should not be same", ugi11, url22); awaitCacheEmptyDueToExpiration(); ugi12 = ugiProvider1.ugi(); url22 = ugiProvider2.ugi(); String msg = "With cache eviction, two UGIs returned" + " by the same token should not be same"; Assert.assertNotEquals(msg, ugi11, ugi12); Assert.assertNotEquals(msg, url21, url22); Assert.assertNotEquals( "With UGI cache, two UGIs for the different token should not be same", ugi11, url22); }
@Test public void testUGICacheInSecure() throws Exception { String uri1 = WebHdfsFileSystem.PATH_PREFIX + PATH + "?op=OPEN" + Param.toSortedString("&", new OffsetParam((long) OFFSET), new LengthParam((long) LENGTH), new UserParam("root")); String uri2 = WebHdfsFileSystem.PATH_PREFIX + PATH + "?op=OPEN" + Param.toSortedString("&", new OffsetParam((long) OFFSET), new LengthParam((long) LENGTH), new UserParam("hdfs")); DataNodeUGIProvider ugiProvider1 = new DataNodeUGIProvider( new ParameterParser(new QueryStringDecoder(URI.create(uri1)), conf)); UserGroupInformation ugi11 = ugiProvider1.ugi(); UserGroupInformation ugi12 = ugiProvider1.ugi(); Assert.assertEquals( "With UGI cache, two UGIs for the same user should be same", ugi11, ugi12); DataNodeUGIProvider ugiProvider2 = new DataNodeUGIProvider( new ParameterParser(new QueryStringDecoder(URI.create(uri2)), conf)); UserGroupInformation url21 = ugiProvider2.ugi(); UserGroupInformation url22 = ugiProvider2.ugi(); Assert.assertEquals( "With UGI cache, two UGIs for the same user should be same", url21, url22); Assert.assertNotEquals( "With UGI cache, two UGIs for the different user should not be same", ugi11, url22); awaitCacheEmptyDueToExpiration(); ugi12 = ugiProvider1.ugi(); url22 = ugiProvider2.ugi(); String msg = "With cache eviction, two UGIs returned by" + " the same user should not be same"; Assert.assertNotEquals(msg, ugi11, ugi12); Assert.assertNotEquals(msg, url21, url22); Assert.assertNotEquals( "With UGI cache, two UGIs for the different user should not be same", ugi11, url22); }