Java 类org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode.ErrorSimulator 实例源码

项目:hadoop-on-lustre    文件:TransferFsImage.java   
/**
 * A server-side method to respond to a getfile http request
 * Copies the contents of the local file into the output stream.
 */
static void getFileServer(OutputStream outstream, File localfile) 
  throws IOException {
  byte buf[] = new byte[BUFFER_SIZE];
  FileInputStream infile = null;
  try {
    infile = new FileInputStream(localfile);
    if (ErrorSimulator.getErrorSimulation(2)
        && localfile.getAbsolutePath().contains("secondary")) {
      // throw exception only when the secondary sends its image
      throw new IOException("If this exception is not caught by the " +
          "name-node fs image will be truncated.");
    }
    int num = 1;
    while (num > 0) {
      num = infile.read(buf);
      if (num <= 0) {
        break;
      }
      outstream.write(buf, 0, num);
    }
  } finally {
    if (infile != null) {
      infile.close();
    }
  }
}
项目:hadoop-0.20    文件:TransferFsImage.java   
/**
 * A server-side method to respond to a getfile http request
 * Copies the contents of the local file into the output stream.
 */
static void getFileServer(OutputStream outstream, File localfile) 
  throws IOException {
  byte buf[] = new byte[BUFFER_SIZE];
  FileInputStream infile = null;
  try {
    infile = new FileInputStream(localfile);
    if (ErrorSimulator.getErrorSimulation(2)
        && localfile.getAbsolutePath().contains("secondary")) {
      // throw exception only when the secondary sends its image
      throw new IOException("If this exception is not caught by the " +
          "name-node fs image will be truncated.");
    }

    if (ErrorSimulator.getErrorSimulation(3)
        && localfile.getAbsolutePath().contains("fsimage")) {
        // Test sending image shorter than localfile
        long len = localfile.length();
        buf = new byte[(int)Math.min(len/2, BUFFER_SIZE)];
        // This will read at most half of the image
        // and the rest of the image will be sent over the wire
        infile.read(buf);
    }
    int num = 1;
    while (num > 0) {
      num = infile.read(buf);
      if (num <= 0) {
        break;
      }
      outstream.write(buf, 0, num);
    }
  } finally {
    if (infile != null) {
      infile.close();
    }
  }
}
项目:hortonworks-extension    文件:TransferFsImage.java   
/**
 * A server-side method to respond to a getfile http request
 * Copies the contents of the local file into the output stream.
 */
static void getFileServer(OutputStream outstream, File localfile) 
  throws IOException {
  byte buf[] = new byte[BUFFER_SIZE];
  FileInputStream infile = null;
  try {
    infile = new FileInputStream(localfile);
    if (ErrorSimulator.getErrorSimulation(2)
        && localfile.getAbsolutePath().contains("secondary")) {
      // throw exception only when the secondary sends its image
      throw new IOException("If this exception is not caught by the " +
          "name-node fs image will be truncated.");
    }
    int num = 1;
    while (num > 0) {
      num = infile.read(buf);
      if (num <= 0) {
        break;
      }
      outstream.write(buf, 0, num);
    }
  } finally {
    if (infile != null) {
      infile.close();
    }
  }
}
项目:hortonworks-extension    文件:TransferFsImage.java   
/**
 * A server-side method to respond to a getfile http request
 * Copies the contents of the local file into the output stream.
 */
static void getFileServer(OutputStream outstream, File localfile) 
  throws IOException {
  byte buf[] = new byte[BUFFER_SIZE];
  FileInputStream infile = null;
  try {
    infile = new FileInputStream(localfile);
    if (ErrorSimulator.getErrorSimulation(2)
        && localfile.getAbsolutePath().contains("secondary")) {
      // throw exception only when the secondary sends its image
      throw new IOException("If this exception is not caught by the " +
          "name-node fs image will be truncated.");
    }
    int num = 1;
    while (num > 0) {
      num = infile.read(buf);
      if (num <= 0) {
        break;
      }
      outstream.write(buf, 0, num);
    }
  } finally {
    if (infile != null) {
      infile.close();
    }
  }
}
项目:hadoop-gpu    文件:TransferFsImage.java   
/**
 * A server-side method to respond to a getfile http request
 * Copies the contents of the local file into the output stream.
 */
static void getFileServer(OutputStream outstream, File localfile) 
  throws IOException {
  byte buf[] = new byte[BUFFER_SIZE];
  FileInputStream infile = null;
  try {
    infile = new FileInputStream(localfile);
    if (ErrorSimulator.getErrorSimulation(2)
        && localfile.getAbsolutePath().contains("secondary")) {
      // throw exception only when the secondary sends its image
      throw new IOException("If this exception is not caught by the " +
          "name-node fs image will be truncated.");
    }
    int num = 1;
    while (num > 0) {
      num = infile.read(buf);
      if (num <= 0) {
        break;
      }
      outstream.write(buf, 0, num);
    }
  } finally {
    if (infile != null) {
      infile.close();
    }
  }
}
项目:hadoop-on-lustre    文件:TestCheckpoint.java   
/**
 * Simulate a secondary node failure to transfer image
 * back to the name-node.
 * Used to truncate primary fsimage file.
 */
void testSecondaryFailsToReturnImage(Configuration conf)
  throws IOException {
  System.out.println("Starting testSecondaryFailsToReturnImage");
  Path file1 = new Path("checkpointRI.dat");
  MiniDFSCluster cluster = new MiniDFSCluster(conf, numDatanodes, 
                                              false, null);
  cluster.waitActive();
  FileSystem fileSys = cluster.getFileSystem();
  FSImage image = cluster.getNameNode().getFSImage();
  try {
    assertTrue(!fileSys.exists(file1));
    StorageDirectory sd = null;
    for (Iterator<StorageDirectory> it = 
              image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();)
       sd = it.next();
    assertTrue(sd != null);
    long fsimageLength = FSImage.getImageFile(sd, NameNodeFile.IMAGE).length();
    //
    // Make the checkpoint
    //
    SecondaryNameNode secondary = startSecondaryNameNode(conf);
    ErrorSimulator.setErrorSimulation(2);

    try {
      secondary.doCheckpoint();  // this should fail
      assertTrue(false);
    } catch (IOException e) {
      System.out.println("testSecondaryFailsToReturnImage: doCheckpoint() " +
          "failed predictably - " + e);
    }
    ErrorSimulator.clearErrorSimulation(2);

    // Verify that image file sizes did not change.
    for (Iterator<StorageDirectory> it = 
            image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();) {
      assertTrue(FSImage.getImageFile(it.next(), 
                              NameNodeFile.IMAGE).length() == fsimageLength);
    }

    secondary.shutdown();
  } finally {
    fileSys.close();
    cluster.shutdown();
  }
}
项目:RDFS    文件:TestCheckpoint.java   
/**
 * Simulate a secondary node failure to transfer image
 * back to the name-node.
 * Used to truncate primary fsimage file.
 */
void testSecondaryFailsToReturnImage(Configuration conf)
  throws IOException {
  System.out.println("Starting testSecondaryFailsToReturnImage");
  Path file1 = new Path("checkpointRI.dat");
  MiniDFSCluster cluster = new MiniDFSCluster(conf, numDatanodes, 
                                              false, null);
  cluster.waitActive();
  FileSystem fileSys = cluster.getFileSystem();
  FSImage image = cluster.getNameNode().getFSImage();
  try {
    assertTrue(!fileSys.exists(file1));
    StorageDirectory sd = null;
    for (Iterator<StorageDirectory> it = 
              image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();)
       sd = it.next();
    assertTrue(sd != null);
    long fsimageLength = FSImage.getImageFile(sd, NameNodeFile.IMAGE).length();
    //
    // Make the checkpoint
    //
    SecondaryNameNode secondary = startSecondaryNameNode(conf);
    ErrorSimulator.setErrorSimulation(2);

    try {
      secondary.doCheckpoint();  // this should fail
      assertTrue(false);
    } catch (IOException e) {
      System.out.println("testSecondaryFailsToReturnImage: doCheckpoint() " +
          "failed predictably - " + e);
    }
    ErrorSimulator.clearErrorSimulation(2);

    // Verify that image file sizes did not change.
    for (Iterator<StorageDirectory> it = 
            image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();) {
      assertTrue(FSImage.getImageFile(it.next(), 
                              NameNodeFile.IMAGE).length() == fsimageLength);
    }

    secondary.shutdown();
  } finally {
    fileSys.close();
    cluster.shutdown();
  }
}
项目:RDFS    文件:TestCheckpoint.java   
/**
 * Simulate namenode failing to send the whole file
 * secondary namenode sometimes assumed it received all of it
 */
@SuppressWarnings("deprecation")
void testNameNodeImageSendFail(Configuration conf)
  throws IOException {
  System.out.println("Starting testNameNodeImageSendFail");
  Path file1 = new Path("checkpointww.dat");
  MiniDFSCluster cluster = new MiniDFSCluster(conf, numDatanodes, 
                                              false, null);
  cluster.waitActive();
  FileSystem fileSys = cluster.getFileSystem();
  try {
    assertTrue(!fileSys.exists(file1));
    //
    // Make the checkpoint fail after rolling the edit log.
    //
    SecondaryNameNode secondary = startSecondaryNameNode(conf);
    ErrorSimulator.setErrorSimulation(3);

    try {
      secondary.doCheckpoint();  // this should fail
      fail("Did not get expected exception");
    } catch (IOException e) {
      // We only sent part of the image. Have to trigger this exception
      assertTrue(e.getMessage().contains("is not of the advertised size"));
    }
    ErrorSimulator.clearErrorSimulation(3);
    secondary.shutdown(); // secondary namenode crash!

    // start new instance of secondary and verify that 
    // a new rollEditLog suceedes inspite of the fact that 
    // edits.new already exists.
    //
    secondary = startSecondaryNameNode(conf);
    secondary.doCheckpoint();  // this should work correctly
    secondary.shutdown();

    //
    // Create a new file
    //
    writeFile(fileSys, file1, replication);
    checkFile(fileSys, file1, replication);
  } finally {
    fileSys.close();
    cluster.shutdown();
  }
}
项目:RDFS    文件:TransferFsImage.java   
/**
 * A server-side method to respond to a getfile http request
 * Copies the contents of the local file into the output stream.
 */
static void getFileServer(OutputStream outstream, File localfile, DataTransferThrottler throttler) 
  throws IOException {
  byte buf[] = new byte[BUFFER_SIZE];
  FileInputStream infile = null;
  long totalReads = 0, totalSends = 0;
  try {
    infile = new FileInputStream(localfile);
    if (ErrorSimulator.getErrorSimulation(2)
        && localfile.getAbsolutePath().contains("secondary")) {
      // throw exception only when the secondary sends its image
      throw new IOException("If this exception is not caught by the " +
          "name-node fs image will be truncated.");
    }

    if (ErrorSimulator.getErrorSimulation(3)
        && localfile.getAbsolutePath().contains("fsimage")) {
        // Test sending image shorter than localfile
        long len = localfile.length();
        buf = new byte[(int)Math.min(len/2, BUFFER_SIZE)];
        // This will read at most half of the image
        // and the rest of the image will be sent over the wire
        infile.read(buf);
    }
    int num = 1;
    while (num > 0) {
      long startRead = System.currentTimeMillis();
      num = infile.read(buf);
      if (num <= 0) {
        break;
      }
      outstream.write(buf, 0, num);
      if (throttler != null) {
        throttler.throttle(num);
      }
    }
  } finally {
    if (infile != null) {
      infile.close();
    }
  }
}
项目:hadoop-0.20    文件:TestCheckpoint.java   
/**
 * Simulate a secondary node failure to transfer image
 * back to the name-node.
 * Used to truncate primary fsimage file.
 */
void testSecondaryFailsToReturnImage(Configuration conf)
  throws IOException {
  System.out.println("Starting testSecondaryFailsToReturnImage");
  Path file1 = new Path("checkpointRI.dat");
  MiniDFSCluster cluster = new MiniDFSCluster(conf, numDatanodes, 
                                              false, null);
  cluster.waitActive();
  FileSystem fileSys = cluster.getFileSystem();
  FSImage image = cluster.getNameNode().getFSImage();
  try {
    assertTrue(!fileSys.exists(file1));
    StorageDirectory sd = null;
    for (Iterator<StorageDirectory> it = 
              image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();)
       sd = it.next();
    assertTrue(sd != null);
    long fsimageLength = FSImage.getImageFile(sd, NameNodeFile.IMAGE).length();
    //
    // Make the checkpoint
    //
    SecondaryNameNode secondary = startSecondaryNameNode(conf);
    ErrorSimulator.setErrorSimulation(2);

    try {
      secondary.doCheckpoint();  // this should fail
      assertTrue(false);
    } catch (IOException e) {
      System.out.println("testSecondaryFailsToReturnImage: doCheckpoint() " +
          "failed predictably - " + e);
    }
    ErrorSimulator.clearErrorSimulation(2);

    // Verify that image file sizes did not change.
    for (Iterator<StorageDirectory> it = 
            image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();) {
      assertTrue(FSImage.getImageFile(it.next(), 
                              NameNodeFile.IMAGE).length() == fsimageLength);
    }

    secondary.shutdown();
  } finally {
    fileSys.close();
    cluster.shutdown();
  }
}
项目:hadoop-0.20    文件:TestCheckpoint.java   
/**
 * Simulate namenode failing to send the whole file
 * secondary namenode sometimes assumed it received all of it
 */
@SuppressWarnings("deprecation")
void testNameNodeImageSendFail(Configuration conf)
  throws IOException {
  System.out.println("Starting testNameNodeImageSendFail");
  Path file1 = new Path("checkpointww.dat");
  MiniDFSCluster cluster = new MiniDFSCluster(conf, numDatanodes, 
                                              false, null);
  cluster.waitActive();
  FileSystem fileSys = cluster.getFileSystem();
  try {
    assertTrue(!fileSys.exists(file1));
    //
    // Make the checkpoint fail after rolling the edit log.
    //
    SecondaryNameNode secondary = startSecondaryNameNode(conf);
    ErrorSimulator.setErrorSimulation(3);

    try {
      secondary.doCheckpoint();  // this should fail
      fail("Did not get expected exception");
    } catch (IOException e) {
      // We only sent part of the image. Have to trigger this exception
      assertTrue(e.getMessage().contains("is not of the advertised size"));
    }
    ErrorSimulator.clearErrorSimulation(3);
    secondary.shutdown(); // secondary namenode crash!

    // start new instance of secondary and verify that 
    // a new rollEditLog suceedes inspite of the fact that 
    // edits.new already exists.
    //
    secondary = startSecondaryNameNode(conf);
    secondary.doCheckpoint();  // this should work correctly
    secondary.shutdown();

    //
    // Create a new file
    //
    writeFile(fileSys, file1, replication);
    checkFile(fileSys, file1, replication);
  } finally {
    fileSys.close();
    cluster.shutdown();
  }
}
项目:hortonworks-extension    文件:TestCheckpoint.java   
/**
 * Simulate a secondary node failure to transfer image
 * back to the name-node.
 * Used to truncate primary fsimage file.
 */
void testSecondaryFailsToReturnImage(Configuration conf)
  throws IOException {
  System.out.println("Starting testSecondaryFailsToReturnImage");
  Path file1 = new Path("checkpointRI.dat");
  MiniDFSCluster cluster = new MiniDFSCluster(conf, numDatanodes, 
                                              false, null);
  cluster.waitActive();
  FileSystem fileSys = cluster.getFileSystem();
  FSImage image = cluster.getNameNode().getFSImage();
  try {
    assertTrue(!fileSys.exists(file1));
    StorageDirectory sd = null;
    for (Iterator<StorageDirectory> it = 
              image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();)
       sd = it.next();
    assertTrue(sd != null);
    long fsimageLength = FSImage.getImageFile(sd, NameNodeFile.IMAGE).length();
    //
    // Make the checkpoint
    //
    SecondaryNameNode secondary = startSecondaryNameNode(conf);
    ErrorSimulator.setErrorSimulation(2);

    try {
      secondary.doCheckpoint();  // this should fail
      assertTrue(false);
    } catch (IOException e) {
      System.out.println("testSecondaryFailsToReturnImage: doCheckpoint() " +
          "failed predictably - " + e);
    }
    ErrorSimulator.clearErrorSimulation(2);

    // Verify that image file sizes did not change.
    for (Iterator<StorageDirectory> it = 
            image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();) {
      assertTrue(FSImage.getImageFile(it.next(), 
                              NameNodeFile.IMAGE).length() == fsimageLength);
    }

    secondary.shutdown();
  } finally {
    fileSys.close();
    cluster.shutdown();
  }
}
项目:hortonworks-extension    文件:TestCheckpoint.java   
/**
 * Simulate a secondary node failure to transfer image
 * back to the name-node.
 * Used to truncate primary fsimage file.
 */
void testSecondaryFailsToReturnImage(Configuration conf)
  throws IOException {
  System.out.println("Starting testSecondaryFailsToReturnImage");
  Path file1 = new Path("checkpointRI.dat");
  MiniDFSCluster cluster = new MiniDFSCluster(conf, numDatanodes, 
                                              false, null);
  cluster.waitActive();
  FileSystem fileSys = cluster.getFileSystem();
  FSImage image = cluster.getNameNode().getFSImage();
  try {
    assertTrue(!fileSys.exists(file1));
    StorageDirectory sd = null;
    for (Iterator<StorageDirectory> it = 
              image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();)
       sd = it.next();
    assertTrue(sd != null);
    long fsimageLength = FSImage.getImageFile(sd, NameNodeFile.IMAGE).length();
    //
    // Make the checkpoint
    //
    SecondaryNameNode secondary = startSecondaryNameNode(conf);
    ErrorSimulator.setErrorSimulation(2);

    try {
      secondary.doCheckpoint();  // this should fail
      assertTrue(false);
    } catch (IOException e) {
      System.out.println("testSecondaryFailsToReturnImage: doCheckpoint() " +
          "failed predictably - " + e);
    }
    ErrorSimulator.clearErrorSimulation(2);

    // Verify that image file sizes did not change.
    for (Iterator<StorageDirectory> it = 
            image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();) {
      assertTrue(FSImage.getImageFile(it.next(), 
                              NameNodeFile.IMAGE).length() == fsimageLength);
    }

    secondary.shutdown();
  } finally {
    fileSys.close();
    cluster.shutdown();
  }
}
项目:hadoop-gpu    文件:TestCheckpoint.java   
/**
 * Simulate a secondary node failure to transfer image
 * back to the name-node.
 * Used to truncate primary fsimage file.
 */
void testSecondaryFailsToReturnImage(Configuration conf)
  throws IOException {
  System.out.println("Starting testSecondaryFailsToReturnImage");
  Path file1 = new Path("checkpointRI.dat");
  MiniDFSCluster cluster = new MiniDFSCluster(conf, numDatanodes, 
                                              false, null);
  cluster.waitActive();
  FileSystem fileSys = cluster.getFileSystem();
  FSImage image = cluster.getNameNode().getFSImage();
  try {
    assertTrue(!fileSys.exists(file1));
    StorageDirectory sd = null;
    for (Iterator<StorageDirectory> it = 
              image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();)
       sd = it.next();
    assertTrue(sd != null);
    long fsimageLength = FSImage.getImageFile(sd, NameNodeFile.IMAGE).length();
    //
    // Make the checkpoint
    //
    SecondaryNameNode secondary = startSecondaryNameNode(conf);
    ErrorSimulator.setErrorSimulation(2);

    try {
      secondary.doCheckpoint();  // this should fail
      assertTrue(false);
    } catch (IOException e) {
      System.out.println("testSecondaryFailsToReturnImage: doCheckpoint() " +
          "failed predictably - " + e);
    }
    ErrorSimulator.clearErrorSimulation(2);

    // Verify that image file sizes did not change.
    for (Iterator<StorageDirectory> it = 
            image.dirIterator(NameNodeDirType.IMAGE); it.hasNext();) {
      assertTrue(FSImage.getImageFile(it.next(), 
                              NameNodeFile.IMAGE).length() == fsimageLength);
    }

    secondary.shutdown();
  } finally {
    fileSys.close();
    cluster.shutdown();
  }
}