1. 包列表
  2. 类列表
  3. org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat

Java 类org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat 实例源码

项目：big-c    文件：MutiWordcount.java    
public static void main(String[] args) throws Exception {

  Configuration conf = new Configuration();

  String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

  long maxInputSplitSize = Long.parseLong(otherArgs[otherArgs.length-1]);
  long minInputSplitSize = Long.parseLong(otherArgs[otherArgs.length-2]);

  //added by wei
  conf.setLong(CombineTextInputFormat.SPLIT_MINSIZE_PERNODE, minInputSplitSize);
  conf.setLong(CombineTextInputFormat.SPLIT_MINSIZE_PERRACK, minInputSplitSize);
  //----


  if (otherArgs.length < 2) {
    System.err.println("Usage: wordcount <in> [<in>...] <out>");
    System.exit(2);
  }
  Job job = new Job(conf, "mulword count");
  job.setJarByClass(WordCount.class);
  job.setMapperClass(TokenizerMapper.class);
  job.setCombinerClass(IntSumReducer.class);
  job.setReducerClass(IntSumReducer.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(IntWritable.class);

  //added by wei
  job.setInputFormatClass(CombineTextInputFormat.class);
  CombineTextInputFormat.setMaxInputSplitSize(job, maxInputSplitSize);
  //----

  for (int i = 0; i < otherArgs.length - 3; ++i) {
    FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
  }
  FileOutputFormat.setOutputPath(job,
    new Path(otherArgs[otherArgs.length - 3]));
  System.exit(job.waitForCompletion(true) ? 0 : 1);
}
项目：FlexMap    文件：MutiWordcount.java    
public static void main(String[] args) throws Exception {

  Configuration conf = new Configuration();

  String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

  long maxInputSplitSize = Long.parseLong(otherArgs[otherArgs.length-1]);
  long minInputSplitSize = Long.parseLong(otherArgs[otherArgs.length-2]);

  //added by wei
  conf.setLong(CombineTextInputFormat.SPLIT_MINSIZE_PERNODE, minInputSplitSize);
  conf.setLong(CombineTextInputFormat.SPLIT_MINSIZE_PERRACK, minInputSplitSize);
  //----


  if (otherArgs.length < 2) {
    System.err.println("Usage: wordcount <in> [<in>...] <out>");
    System.exit(2);
  }
  Job job = new Job(conf, "mulword count");
  job.setJarByClass(WordCount.class);
  job.setMapperClass(TokenizerMapper.class);
  job.setCombinerClass(IntSumReducer.class);
  job.setReducerClass(IntSumReducer.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(IntWritable.class);

  //added by wei
  job.setInputFormatClass(CombineTextInputFormat.class);
  CombineTextInputFormat.setMaxInputSplitSize(job, maxInputSplitSize);
  //----

  for (int i = 0; i < otherArgs.length - 3; ++i) {
    FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
  }
  FileOutputFormat.setOutputPath(job,
    new Path(otherArgs[otherArgs.length - 3]));
  System.exit(job.waitForCompletion(true) ? 0 : 1);
}
项目：angel    文件：MFLocalExample.java    
public void setConf() throws Exception {
  String inputPath = "../data/exampledata/MFLocalExampleData";
  // Set local deploy mode
  conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
  // Set basic configuration keys
  conf.setBoolean("mapred.mapper.new-api", true);
  conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
  conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);

  // set angel resource parameters #worker, #task, #PS
  conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);

  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");

  // Set trainning data, and save model path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/model");
  conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
  // Set actionType train
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_TRAIN());

  // Set MF algorithm parameters
  conf.set(MLConf.ML_MF_RANK(), "200");
  conf.set(MLConf.ML_EPOCH_NUM(), "8");
  conf.set(MLConf.ML_MF_ROW_BATCH_NUM(), "2");
  conf.set(MLConf.ML_MF_ITEM_NUM(), "1683");
  conf.set(MLConf.ML_MF_LAMBDA(), "0.01");
  conf.set(MLConf.ML_MF_ETA(), "0.0054");
}
项目：angel    文件：GBDTLocalExample.java    
public void setConf() {
  String inputPath = "../data/exampledata/GBDTLocalExampleData/agaricus.txt.train";

  // Feature number of train data
  int featureNum = 127;
  // Number of nonzero features
  int featureNzz = 25;
  // Tree number
  int treeNum = 2;
  // Tree depth
  int treeDepth = 2;
  // Split number
  int splitNum = 10;
  // Feature sample ratio
  double sampleRatio = 1.0;

  // Data format
  String dataFmt = "libsvm";

  // Learning rate
  double learnRate = 0.01;

  // Set basic configuration keys
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");

  conf.setBoolean("mapred.mapper.new-api", true);
  conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);

  // Use local deploy mode and data format
  conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
  conf.set(MLConf.ML_DATA_FORMAT(), String.valueOf(dataFmt));

  // set input, output path
  conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
  conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

  // set angel resource parameters #worker, #task, #PS
  conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);

  // Set GBDT algorithm parameters
  conf.set(MLConf.ML_FEATURE_NUM(), String.valueOf(featureNum));
  conf.set(MLConf.ML_FEATURE_NNZ(), String.valueOf(featureNzz));
  conf.set(MLConf.ML_GBDT_TREE_NUM(), String.valueOf(treeNum));
  conf.set(MLConf.ML_GBDT_TREE_DEPTH(), String.valueOf(treeDepth));
  conf.set(MLConf.ML_GBDT_SPLIT_NUM(), String.valueOf(splitNum));
  conf.set(MLConf.ML_GBDT_SAMPLE_RATIO(), String.valueOf(sampleRatio));
  conf.set(MLConf.ML_LEARN_RATE(), String.valueOf(learnRate));
}
项目：angel    文件：SGDLRLocalExample.java    
public void setConf() {

    // Feature number of train data
    int featureNum = 124;
    // Total iteration number
    int epochNum = 20;
    // Validation sample Ratio
    double vRatio = 0.1;
    // Data format, libsvm or dummy
    String dataFmt = "libsvm";
    // Train batch number per epoch.
    double spRatio = 1.0;
    // Batch number
    int batchNum = 10;

    // Learning rate
    double learnRate = 1.0;
    // Decay of learning rate
    double decay = 0.1;
    // Regularization coefficient
    double reg = 0.2;

    // Set local deploy mode
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");

    // Set basic configuration keys
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);

    // Set data format
    conf.set(MLConf.ML_DATA_FORMAT(), dataFmt);

    //set angel resource parameters #worker, #task, #PS
    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);

    //set sgd LR algorithm parameters #feature #epoch
    conf.set(MLConf.ML_FEATURE_NUM(), String.valueOf(featureNum));
    conf.set(MLConf.ML_EPOCH_NUM(), String.valueOf(epochNum));
    conf.set(MLConf.ML_BATCH_SAMPLE_Ratio(), String.valueOf(spRatio));
    conf.set(MLConf.ML_VALIDATE_RATIO(), String.valueOf(vRatio));
    conf.set(MLConf.ML_LEARN_RATE(), String.valueOf(learnRate));
    conf.set(MLConf.ML_LEARN_DECAY(), String.valueOf(decay));
    conf.set(MLConf.ML_REG_L2(), String.valueOf(reg));
  }
项目：angel    文件：LinearRegLocalExample.java    
public void setConf() {

    // Feature number of train data
    int featureNum = 101;
    // Total iteration number
    int epochNum = 20;
    // Validation sample Ratio
    double vRatio = 0.5;
    // Data format, libsvm or dummy
    String dataFmt = "libsvm";
    // Train batch number per epoch.
    double spRatio = 1;

    // Learning rate
    double learnRate = 0.1;
    // Decay of learning rate
    double decay = 0.01;
    // Regularization coefficient
    double reg = 0;

    // Set local deploy mode
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");

    // Set basic configuration keys
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);

    // Set data format
    conf.set(MLConf.ML_DATA_FORMAT(), dataFmt);

    // set angel resource parameters #worker, #task, #PS
    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);

    // set sgd LR algorithm parameters #feature #epoch
    conf.set(MLConf.ML_FEATURE_NUM(), String.valueOf(featureNum));
    conf.set(MLConf.ML_EPOCH_NUM(), String.valueOf(epochNum));
    conf.set(MLConf.ML_BATCH_SAMPLE_Ratio(), String.valueOf(spRatio));
    conf.set(MLConf.ML_VALIDATE_RATIO(), String.valueOf(vRatio));
    conf.set(MLConf.ML_LEARN_RATE(), String.valueOf(learnRate));
    conf.set(MLConf.ML_LEARN_DECAY(), String.valueOf(decay));
    conf.set(MLConf.ML_REG_L2(), String.valueOf(reg));
  }
项目：angel    文件：QSLRLocalExample.java    
public static Configuration setConf() {
  Configuration conf = new Configuration();

  // Feature number of train data
  int featureNum = 124;
  // Total iteration number
  int epochNum = 20;
  // Learning rate
  double learnRate = 1.0;
  // Regularization coefficient
  double reg = 0.2;

  String inputPath = "./angel-ps/mllib/src/test/data/lr/a9a.train";
  String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
  String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");
  String savePath = LOCAL_FS + TMP_PATH + "/model";

  // Set local deploy mode
  conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");

  // Set basic configuration keys
  conf.setBoolean("mapred.mapper.new-api", true);
  conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
  conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);

  //set angel resource parameters #worker, #task, #PS
  conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);

  //set sgd LR algorithm parameters #feature #epoch
  conf.set(MLConf.ML_FEATURE_NUM(), String.valueOf(featureNum));
  conf.set(MLConf.ML_EPOCH_NUM(), String.valueOf(epochNum));
  conf.set(MLConf.ML_LEARN_RATE(), String.valueOf(learnRate));
  conf.set(MLConf.ML_REG_L2(), String.valueOf(reg));

  // Set input data path
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
  // Set save model path
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, savePath);
  // Set actionType train
  conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_TRAIN());
  conf.set(MLConf.ML_DATA_FORMAT(), "libsvm");
  return conf;
}
项目：angel    文件：SgdLRTest.java    
/**
 * set parameter values of conf
 */
@Before
public void setConf() throws Exception {
  try {
    // Feature number of train data
    int featureNum = 124;
    // Total iteration number
    int epochNum = 5;
    // Validation sample Ratio
    double vRatio = 0.1;
    // Data format, libsvm or dummy
    String dataFmt = "libsvm";
    // Train batch number per epoch.
    double spRatio = 1.0;
    // Batch number
    int batchNum = 10;
    // Model type
    String modelType = String.valueOf(RowType.T_DOUBLE_DENSE);

    // Learning rate
    double learnRate = 1.0;
    // Decay of learning rate
    double decay = 1;
    // Regularization coefficient
    double reg = 0.002;

    // Set local deploy mode
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");

    // Set basic configuration keys
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
    conf.set(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, "true");
    conf.setBoolean(MLConf.ML_INDEX_GET_ENABLE(), true);
    // Set data format
    conf.set(MLConf.ML_DATA_FORMAT(), dataFmt);

    //set angel resource parameters #worker, #task, #PS
    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);

    //set sgd LR algorithm parameters #feature #epoch
    conf.set(MLConf.LR_MODEL_TYPE(), modelType);
    conf.set(MLConf.ML_FEATURE_NUM(), String.valueOf(featureNum));
    conf.set(MLConf.ML_EPOCH_NUM(), String.valueOf(epochNum));
    conf.set(MLConf.ML_BATCH_SAMPLE_Ratio(), String.valueOf(spRatio));
    conf.set(MLConf.ML_VALIDATE_RATIO(), String.valueOf(vRatio));
    conf.set(MLConf.ML_LEARN_RATE(), String.valueOf(learnRate));
    conf.set(MLConf.ML_LEARN_DECAY(), String.valueOf(decay));
    conf.set(MLConf.ML_REG_L2(), String.valueOf(reg));
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：GBDTTest.java    
@Before public void setup() throws Exception {
  try {

    conf.set(MLConf.ML_GBDT_TASK_TYPE(), "classification");

    // Feature number of train data
    int featureNum = 127;
    // Number of nonzero features
    int featureNzz = 25;
    // Tree number
    int treeNum = 5;
    // Tree depth
    int treeDepth = 3;
    // Split number
    int splitNum = 10;
    // Feature sample ratio
    double sampleRatio = 1.0;

    // Data format
    String dataFmt = "libsvm";

    // Learning rate
    double learnRate = 0.01;

    // Set basic configuration keys
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);

    // Use local deploy mode and dummy data spliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");

    // set input, output path
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/LOG/log");

    //set angel resource parameters #worker, #task, #PS
    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);

    // Set GBDT algorithm parameters
    conf.set(MLConf.ML_DATA_FORMAT(), dataFmt);
    conf.set(MLConf.ML_FEATURE_NUM(), String.valueOf(featureNum));
    conf.set(MLConf.ML_FEATURE_NNZ(), String.valueOf(featureNzz));
    conf.set(MLConf.ML_GBDT_TREE_NUM(), String.valueOf(treeNum));
    conf.set(MLConf.ML_GBDT_TREE_DEPTH(), String.valueOf(treeDepth));
    conf.set(MLConf.ML_GBDT_SPLIT_NUM(), String.valueOf(splitNum));
    conf.set(MLConf.ML_GBDT_SAMPLE_RATIO(), String.valueOf(sampleRatio));
    conf.set(MLConf.ML_LEARN_RATE(), String.valueOf(learnRate));
    conf.set(MLConf.ML_GBDT_CATE_FEAT(),
        "0:2,1:2,2:2,3:2,4:2,5:2,6:2,7:2,8:2,9:2,10:2,11:2,12:2,13:2,14:2,15:2,16:2,17:2,18:2,19:2,20:2,21:2,22:2,23:2,24:2,25:2,26:2,27:2,28:2,29:2,30:2,31:2,32:2,33:2,34:2,35:2,36:2,37:2,38:2,39:2,40:2,41:2,42:2,43:2,44:2,45:2,46:2,47:2,48:2,49:2,50:2,51:2,52:2,53:2,54:2,55:2,56:2,57:2,58:2,59:2,60:2,61:2,62:2,63:2,64:2,65:2,66:2,67:2,68:2,69:2,70:2,71:2,72:2,73:2,74:2,75:2,76:2,77:2,78:2,79:2,80:2,81:2,82:2,83:2,84:2,85:2,86:2,87:2,88:2,89:2,90:2,91:2,92:2,93:2,94:2,95:2,96:2,97:2,98:2,99:2,100:2,101:2,102:2,103:2,104:2,105:2,106:2,107:2,108:2,109:2,110:2,111:2,112:2,113:2,114:2,115:2,116:2,117:2,118:2,119:2,120:2,121:2,122:2,123:2,124:2,125:2,126:2");
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：AlgoMetricsTest.java    
@Before
public void setup() throws Exception {
  try{
    // set basic configuration keys
    Configuration conf = new Configuration();
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);

    // use local deploy mode and dummy dataspliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);

    // get a angel client
    angelClient = AngelClientFactory.get(conf);

    // add matrix
    MatrixContext mMatrix = new MatrixContext();
    mMatrix.setName("w1");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(100000);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(50000);
    mMatrix.setRowType(RowType.T_INT_DENSE);
    mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
    angelClient.addMatrix(mMatrix);

    angelClient.startPSServer();
    angelClient.runTask(MetricTestTask.class);
    Thread.sleep(5000);
    task0Id = new TaskId(0);
    task1Id = new TaskId(1);
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：matrixfactorizationTest.java    
@Test
public void testMF() throws Exception {
  try {
    String inputPath = "./src/test/data/recommendation/MovieLensDataSet";

    // Set local deploy mode
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    // Set basic configuration keys
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);

    //set angel resource parameters #worker, #task, #PS
    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);

    String LOCAL_FS = LocalFileSystem.DEFAULT_FS;
    String TMP_PATH = System.getProperty("java.io.tmpdir", "/tmp");

    // Set trainning data, save model, log path
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, inputPath);
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/model");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/Log/log");
    conf.set(MLConf.ML_MF_USER_OUTPUT_PATH(), LOCAL_FS + TMP_PATH + "/usermodel");
    // Set actionType train
    conf.set(AngelConf.ANGEL_ACTION_TYPE, MLConf.ANGEL_ML_TRAIN());

    // Set MF algorithm parameters
    conf.set(MLConf.ML_MF_RANK(), "200");
    conf.set(MLConf.ML_EPOCH_NUM(), "5");
    conf.set(MLConf.ML_MF_ROW_BATCH_NUM(), "2");
    conf.set(MLConf.ML_MF_ITEM_NUM(), "1683");
    conf.set(MLConf.ML_MF_LAMBDA(), "0.01");
    conf.set(MLConf.ML_MF_ETA(), "0.0005");

    MatrixFactorizationRunner runner = new MatrixFactorizationRunner();
    runner.train(conf);
  } catch (Exception x) {
    LOG.error("run testMF failed ", x);
    throw x;
  }
}
项目：angel    文件：FMTest.java    
/**
 * set parameter values of conf
 */
@Before
public void setConf() {

  // Feature number of train data
  int featureNum = 236;
  // Total iteration number
  int epochNum = 5;
  // Rank
  int rank = 5;
  // Regularization parameters
  double reg0 = 0.0;
  double reg1 = 0.0;
  double reg2 = 0.001;
  // Learn rage
  double lr = 0.001;
  double stev = 0.1;

  // Set local deploy mode
  conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");

  // Set basic configuration keys
  conf.setBoolean("mapred.mapper.new-api", true);
  conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
  conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);

  //set angel resource parameters #worker, #task, #PS
  conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);

  //set FM algorithm parameters #feature #epoch
  conf.set(MLConf.ML_FEATURE_NUM(), String.valueOf(featureNum));
  conf.set(MLConf.ML_EPOCH_NUM(), String.valueOf(epochNum));
  conf.set(MLConf.ML_FM_RANK(), String.valueOf(rank));
  conf.set(MLConf.ML_LEARN_RATE(), String.valueOf(lr));
  conf.set(MLConf.ML_FM_REG0(), String.valueOf(reg0));
  conf.set(MLConf.ML_FM_REG1(), String.valueOf(reg1));
  conf.set(MLConf.ML_FM_REG2(), String.valueOf(reg2));
  conf.set(MLConf.ML_FM_V_STDDEV(), String.valueOf(stev));
}
项目：angel    文件：KmeansTest.java    
@Before
public void setup() throws Exception {
  try {
    String dataFmt = "libsvm";

    // Cluster center number
    int centerNum = 3;
    // Feature number of train data
    int featureNum = 4;
    // Total iteration number
    int epochNum = 5;
    // Sample ratio per mini-batch
    double spratio = 1.0;
    // C
    double c = 0.15;

    // Set local deploy mode
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");

    // Set basic configuration keys
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);

    //set angel resource parameters #worker, #task, #PS
    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);

    //set Kmeans algorithm parameters #cluster #feature #epoch
    conf.set(MLConf.KMEANS_CENTER_NUM(), String.valueOf(centerNum));
    conf.set(MLConf.ML_FEATURE_NUM(), String.valueOf(featureNum));
    conf.set(MLConf.ML_EPOCH_NUM(), String.valueOf(epochNum));
    conf.set(MLConf.KMEANS_SAMPLE_RATIO_PERBATCH(), String.valueOf(spratio));
    conf.set(MLConf.kMEANS_C(), String.valueOf(c));

    // Set data format
    conf.set(MLConf.ML_DATA_FORMAT(), dataFmt);
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：MLRTest.java    
/**
 * set parameter values of conf
 */
@Before public void setConf() throws Exception {
  try {
    // Feature number of train data
    int featureNum = 124;
    // Total iteration number
    int epochNum = 5;
    // Validation sample Ratio
    double vRatio = 0.1;
    // Data format, libsvm or dummy
    String dataFmt = "libsvm";
    // Train batch number per epoch.
    double spRatio = 1.0;
    // Batch number
    int batchNum = 10;

    // Learning rate
    double learnRate = 1.0;
    // Decay of learning rate
    double decay = 0.1;
    // Regularization coefficient
    double reg = 0.02;
    int rank = 5;
    double vInit = 0.1;

    // Set local deploy mode
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");

    // Set basic configuration keys
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);

    // Set data format
    conf.set(MLConf.ML_DATA_FORMAT(), dataFmt);

    //set angel resource parameters #worker, #task, #PS
    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);

    //set MLR algorithm parameters #feature #epoch
    conf.set(MLConf.ML_FEATURE_NUM(), String.valueOf(featureNum));
    conf.set(MLConf.ML_EPOCH_NUM(), String.valueOf(epochNum));
    conf.set(MLConf.ML_BATCH_SAMPLE_Ratio(), String.valueOf(spRatio));
    conf.set(MLConf.ML_VALIDATE_RATIO(), String.valueOf(vRatio));
    conf.set(MLConf.ML_LEARN_RATE(), String.valueOf(learnRate));
    conf.set(MLConf.ML_LEARN_DECAY(), String.valueOf(decay));
    conf.set(MLConf.ML_REG_L2(), String.valueOf(reg));
    conf.set(MLConf.ML_MLR_RANK(), String.valueOf(rank));
    conf.set(MLConf.ML_MLR_V_INIT(), String.valueOf(vInit));
    conf.set(MLConf.ML_SGD_BATCH_NUM(), String.valueOf(batchNum));
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：LinearRegTest.java    
/**
 * set parameter values of conf
 */
@Before public void setConf() throws Exception {
  try {
    // Feature number of train data
    int featureNum = 101;
    // Total iteration number
    int epochNum = 5;
    // Validation sample Ratio
    double vRatio = 0.5;
    // Data format, libsvm or dummy
    String dataFmt = "libsvm";
    // Train batch number per epoch.
    double spRatio = 1;

    // Learning rate
    double learnRate = 0.01;
    // Decay of learning rate
    double decay = 0.1;
    // Regularization coefficient
    double reg = 0;

    // Set local deploy mode
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");

    // Set basic configuration keys
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);

    // Set data format
    conf.set(MLConf.ML_DATA_FORMAT(), dataFmt);

    //set angel resource parameters #worker, #task, #PS
    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);

    //set sgd LR algorithm parameters #feature #epoch
    conf.set(MLConf.ML_FEATURE_NUM(), String.valueOf(featureNum));
    conf.set(MLConf.ML_EPOCH_NUM(), String.valueOf(epochNum));
    conf.set(MLConf.ML_BATCH_SAMPLE_Ratio(), String.valueOf(spRatio));
    conf.set(MLConf.ML_VALIDATE_RATIO(), String.valueOf(vRatio));
    conf.set(MLConf.ML_LEARN_RATE(), String.valueOf(learnRate));
    conf.set(MLConf.ML_LEARN_DECAY(), String.valueOf(decay));
    conf.set(MLConf.ML_REG_L2(), String.valueOf(reg));
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：LocalClusterHelper.java    
public static void setup() throws Exception {
  // set basic configuration keys
  Configuration conf = new Configuration();
  conf.setBoolean("mapred.mapper.new-api", true);
  conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
  conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

  // use local deploy mode and dummy data spliter
  conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
  conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
  conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
  conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

  conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);

  // get a angel client
  angelClient = AngelClientFactory.get(conf);

  // add matrix
  MatrixContext mMatrix = new MatrixContext();
  mMatrix.setName("w1");
  mMatrix.setRowNum(10);
  mMatrix.setColNum(1000);
  mMatrix.setMaxRowNumInBlock(10);
  mMatrix.setMaxColNumInBlock(500);
  mMatrix.setRowType(RowType.T_INT_DENSE);
  mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
  mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
  mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
  mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
  angelClient.addMatrix(mMatrix);

  MatrixContext mMatrix2 = new MatrixContext();
  mMatrix2.setName("w2");
  mMatrix2.setRowNum(10);
  mMatrix2.setColNum(100);
  mMatrix2.setMaxRowNumInBlock(5);
  mMatrix2.setMaxColNumInBlock(50);
  mMatrix2.setRowType(RowType.T_DOUBLE_DENSE);
  mMatrix2.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
  mMatrix2.set(MatrixConf.MATRIX_HOGWILD, "false");
  mMatrix2.set(MatrixConf.MATRIX_AVERAGE, "false");
  mMatrix2.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
  angelClient.addMatrix(mMatrix2);

  angelClient.startPSServer();
  angelClient.run();
  Thread.sleep(2 * 1000);
  group0Id = new WorkerGroupId(0);
  worker0Id = new WorkerId(group0Id, 0);
  worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
  task0Id = new TaskId(0);
  task1Id = new TaskId(1);
  psId = new ParameterServerId(0);
  psAttempt0Id = new PSAttemptId(psId, 0);
}
项目：angel    文件：IndexGetFuncTest.java    
@Before
public void setup() throws Exception {
  // set basic configuration keys
  Configuration conf = new Configuration();
  conf.setBoolean("mapred.mapper.new-api", true);
  conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
  conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

  // use local deploy mode and dummy dataspliter
  conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
  conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
  conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
  conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

  conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);

  // get a angel client
  angelClient = AngelClientFactory.get(conf);

  // add dense double matrix
  MatrixContext dMat = new MatrixContext();
  dMat.setName(DENSE_DOUBLE_MAT);
  dMat.setRowNum(1);
  dMat.setColNum(feaNum);
  dMat.setMaxColNumInBlock(feaNum / 3);
  dMat.setRowType(RowType.T_DOUBLE_DENSE);
  angelClient.addMatrix(dMat);

  // add sparse double matrix
  MatrixContext sMat = new MatrixContext();
  sMat.setName(SPARSE_DOUBLE_MAT);
  sMat.setRowNum(1);
  sMat.setColNum(feaNum);
  sMat.setMaxColNumInBlock(feaNum / 3);
  sMat.setRowType(RowType.T_DOUBLE_DENSE);
  angelClient.addMatrix(sMat);

  // add sparse double long key matrix
  MatrixContext lMat = new MatrixContext();
  lMat.setName(SPARSE_DOUBLE_LONG_MAT);
  lMat.setRowNum(1);
  lMat.setColNum(feaNum);
  lMat.setMaxColNumInBlock(feaNum / 3);
  lMat.setRowType(RowType.T_DOUBLE_DENSE);
  angelClient.addMatrix(lMat);

  // Start PS
  angelClient.startPSServer();
  // Start to run application
  angelClient.run();

  Thread.sleep(5000);

  psId = new ParameterServerId(0);
  psAttempt0Id = new PSAttemptId(psId, 0);

  WorkerGroupId workerGroupId = new WorkerGroupId(0);
  workerId = new WorkerId(workerGroupId, 0);
  workerAttempt0Id = new WorkerAttemptId(workerId, 0);
}
项目：angel    文件：ServerPartitionTest.java    
@Test
public void testWriteTo() throws Exception {
  // set basic configuration keys
  conf = new Configuration();
  conf.setBoolean("mapred.mapper.new-api", true);
  conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
  conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

  // use local deploy mode and dummy dataspliter
  conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
  conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
  conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
  conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

  conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);

  // get a angel client
  angelClient = AngelClientFactory.get(conf);

  // add matrix
  MatrixContext mMatrix = new MatrixContext();
  mMatrix.setName("w1");
  mMatrix.setRowNum(1);
  mMatrix.setColNum(100000);
  mMatrix.setMaxRowNumInBlock(1);
  mMatrix.setMaxColNumInBlock(50000);
  mMatrix.setRowType(RowType.T_INT_DENSE);
  mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
  mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
  mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
  mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
  angelClient.addMatrix(mMatrix);

  angelClient.startPSServer();
  angelClient.runTask(DummyTask.class);
  Thread.sleep(5000);

  group0Id = new WorkerGroupId(0);
  worker0Id = new WorkerId(group0Id, 0);
  worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
  task0Id = new TaskId(0);
  task1Id = new TaskId(1);
  psId = new ParameterServerId(0);
  psAttempt0Id = new PSAttemptId(psId, 0);

  DataOutputStream out = new DataOutputStream(new FileOutputStream("data"));
  ByteBuf buf = Unpooled.buffer(16);
  buf.writeDouble(0.00);
  buf.writeDouble(1.00);
  buf.writeDouble(-1.00);
  buf.writeDouble(-2.00);
  buf.writeDouble(-5.00);
  buf.writeDouble(-6.00);
  buf.writeDouble(-7.00);
  buf.writeDouble(-8.00);
  serverPartition.getRow(6).update(RowType.T_DOUBLE_DENSE, buf, 8);
  serverPartition.save(out);
  out.close();
  DataInputStream in = new DataInputStream(new FileInputStream("data"));
  PartitionKey partitionKeyNew = new PartitionKey(2, 1, 1, 2, 8, 10);
  ServerPartition serverPartitionNew =
      new ServerPartition(partitionKeyNew, RowType.T_DOUBLE_DENSE);
  serverPartitionNew.init();
  assertNotEquals(((ServerDenseDoubleRow) serverPartition.getRow(6)).getData(),
      ((ServerDenseDoubleRow) serverPartitionNew.getRow(6)).getData());
  serverPartitionNew.load(in);
  in.close();
  assertEquals(((ServerDenseDoubleRow) serverPartition.getRow(6)).getData(),
      ((ServerDenseDoubleRow) serverPartitionNew.getRow(6)).getData());
  angelClient.stop();
}
项目：angel    文件：ParameterServerServiceTest.java    
@Before
public void setup() throws Exception {
  try {
    // set basic configuration keys
    conf = new Configuration();
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
    conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

    // use local deploy mode and dummy dataspliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);

    // get a angel client
    angelClient = AngelClientFactory.get(conf);

    // add matrix
    MatrixContext mMatrix = new MatrixContext();
    mMatrix.setName("w1");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(100000);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(50000);
    mMatrix.setRowType(RowType.T_INT_DENSE);
    mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
    angelClient.addMatrix(mMatrix);

    mMatrix = new MatrixContext();
    mMatrix.setName("w2");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(100000);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(50000);
    mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
    mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "false");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
    angelClient.addMatrix(mMatrix);

    angelClient.startPSServer();
    angelClient.run();
    Thread.sleep(5000);

    // group0Id = new WorkerGroupId(0);
    // worker0Id = new WorkerId(group0Id, 0);
    // worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
    // task0Id = new TaskId(0);
    // task1Id = new TaskId(1);
    psId = new ParameterServerId(0);
    psAttempt0Id = new PSAttemptId(psId, 0);
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：PSAgentTest.java    
@BeforeClass
public static void setup() throws Exception {
  try {
    // set basic configuration keys
    Configuration conf = new Configuration();
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
    conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

    // use local deploy mode and dummy dataspliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
    // conf.setInt(AngelConf.ANGEL_PREPROCESS_VECTOR_MAXDIM, 10000);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);

    // get a angel client
    angelClient = AngelClientFactory.get(conf);

    // add matrix
    MatrixContext mMatrix = new MatrixContext();
    mMatrix.setName("w1");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(100000);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(50000);
    mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
    mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
    angelClient.addMatrix(mMatrix);

    MatrixContext mMatrix2 = new MatrixContext();
    mMatrix2.setName("w2");
    mMatrix2.setRowNum(1);
    mMatrix2.setColNum(100000);
    mMatrix2.setMaxRowNumInBlock(1);
    mMatrix2.setMaxColNumInBlock(50000);
    mMatrix2.setRowType(RowType.T_DOUBLE_DENSE);
    mMatrix2.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix2.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix2.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix2.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
    angelClient.addMatrix(mMatrix2);

    angelClient.startPSServer();
    angelClient.run();
    Thread.sleep(10000);
    group0Id = new WorkerGroupId(0);
    worker0Id = new WorkerId(group0Id, 0);
    worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
    task0Id = new TaskId(0);
    task1Id = new TaskId(1);
    psId = new ParameterServerId(0);
    psAttempt0Id = new PSAttemptId(psId, 0);
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：MatrixOpLogTest.java    
@Before
public void setup() throws Exception {
  // set basic configuration keys
  Configuration conf = new Configuration();
  conf.setBoolean("mapred.mapper.new-api", true);
  conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
  conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

  // use local deploy mode and dummy dataspliter
  conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
  conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
  conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
  conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

  conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);

  // get a angel client
  angelClient = AngelClientFactory.get(conf);

  // add matrix
  MatrixContext mMatrix = new MatrixContext();
  mMatrix.setName("w1");
  mMatrix.setRowNum(100);
  mMatrix.setColNum(100000);
  mMatrix.setMaxRowNumInBlock(10);
  mMatrix.setMaxColNumInBlock(50000);
  mMatrix.setRowType(RowType.T_INT_DENSE);
  mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
  mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
  mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
  mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
  angelClient.addMatrix(mMatrix);

  angelClient.startPSServer();
  angelClient.run();
  Thread.sleep(5000);
  psId = new ParameterServerId(0);
  psAttempt0Id = new PSAttemptId(psId, 0);
  WorkerGroupId workerGroupId = new WorkerGroupId(0);
  workerId = new WorkerId(workerGroupId, 0);
  workerAttempt0Id = new WorkerAttemptId(workerId, 0);
}
项目：angel    文件：PSFailedReportTest.java    
@Before
public void setup() throws Exception {
  try{
    // set basic configuration keys
    Configuration conf = new Configuration();
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
    conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

    // use local deploy mode and dummy dataspliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 2);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_HA_REPLICATION_NUMBER, 2);
    conf.setInt(AngelConf.ANGEL_PS_HA_PUSH_INTERVAL_MS, 500);
    //conf.setBoolean(AngelConf.ANGEL_PS_HA_USE_EVENT_PUSH, true);
    //conf.setBoolean(AngelConf.ANGEL_PS_HA_PUSH_SYNC, true);

    // get a angel client
    angelClient = AngelClientFactory.get(conf);

    // add matrix
    MatrixContext mMatrix = new MatrixContext();
    mMatrix.setName("w1");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(dim);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(dim / 2);
    mMatrix.setRowType(RowType.T_INT_DENSE);
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    angelClient.addMatrix(mMatrix);

    angelClient.startPSServer();
    angelClient.run();
    Thread.sleep(5000);
    task0Id = new TaskId(0);
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：MasterServiceTest.java    
@Before
public void setup() throws Exception {
  try{
    // set basic configuration keys
    Configuration conf = new Configuration();
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
    conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

    // use local deploy mode and dummy dataspliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);

    // get a angel client
    angelClient = AngelClientFactory.get(conf);

    // add matrix
    MatrixContext mMatrix = new MatrixContext();
    mMatrix.setName("w1");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(100000);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(50000);
    mMatrix.setRowType(RowType.T_INT_DENSE);
    mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
    angelClient.addMatrix(mMatrix);

    MatrixContext mMatrix2 = new MatrixContext();
    mMatrix2.setName("w2");
    mMatrix2.setRowNum(1);
    mMatrix2.setColNum(100000);
    mMatrix2.setMaxRowNumInBlock(1);
    mMatrix2.setMaxColNumInBlock(50000);
    mMatrix2.setRowType(RowType.T_DOUBLE_DENSE);
    mMatrix2.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix2.set(MatrixConf.MATRIX_HOGWILD, "false");
    mMatrix2.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix2.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
    angelClient.addMatrix(mMatrix2);

    angelClient.startPSServer();
    angelClient.run();
    Thread.sleep(5000);
    group0Id = new WorkerGroupId(0);
    worker0Id = new WorkerId(group0Id, 0);
    worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
    task0Id = new TaskId(0);
    task1Id = new TaskId(1);
    psId = new ParameterServerId(0);
    psAttempt0Id = new PSAttemptId(psId, 0);
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：TaskManagerTest.java    
@BeforeClass
public static void setup() throws Exception {
  try {
    // set basic configuration keys
    Configuration conf = new Configuration();
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
    conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

    // use local deploy mode and dummy dataspliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);

    // get a angel client
    angelClient = AngelClientFactory.get(conf);

    // add matrix
    MatrixContext mMatrix = new MatrixContext();
    mMatrix.setName("w1");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(100000);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(50000);
    mMatrix.setRowType(RowType.T_INT_DENSE);
    mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
    angelClient.addMatrix(mMatrix);

    MatrixContext mMatrix2 = new MatrixContext();
    mMatrix2.setName("w2");
    mMatrix2.setRowNum(1);
    mMatrix2.setColNum(100000);
    mMatrix2.setMaxRowNumInBlock(1);
    mMatrix2.setMaxColNumInBlock(50000);
    mMatrix2.setRowType(RowType.T_DOUBLE_DENSE);
    mMatrix2.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix2.set(MatrixConf.MATRIX_HOGWILD, "false");
    mMatrix2.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix2.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
    angelClient.addMatrix(mMatrix2);

    angelClient.startPSServer();
    angelClient.run();
    Thread.sleep(5000);
    group0Id = new WorkerGroupId(0);
    worker0Id = new WorkerId(group0Id, 0);
    worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
    task0Id = new TaskId(0);
    task1Id = new TaskId(1);
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：PeriodHATest.java    
@Before
public void setup() throws Exception {
  try{
    // set basic configuration keys
    Configuration conf = new Configuration();
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
    conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

    // use local deploy mode and dummy dataspliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 2);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_HA_REPLICATION_NUMBER, 2);
    conf.setInt(AngelConf.ANGEL_PS_HA_PUSH_INTERVAL_MS, 1000);
    //conf.setBoolean(AngelConf.ANGEL_PS_HA_USE_EVENT_PUSH, true);
    //conf.setBoolean(AngelConf.ANGEL_PS_HA_PUSH_SYNC, true);

    // get a angel client
    angelClient = AngelClientFactory.get(conf);

    // add matrix
    MatrixContext mMatrix = new MatrixContext();
    mMatrix.setName("w1");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(dim);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(dim / 2);
    mMatrix.setRowType(RowType.T_INT_DENSE);
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    angelClient.addMatrix(mMatrix);

    angelClient.startPSServer();
    angelClient.run();
    Thread.sleep(5000);
    task0Id = new TaskId(0);
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：MatrixMetaManagerTest.java    
@BeforeClass
public static void setup() throws Exception {
  try {
    // set basic configuration keys
    Configuration conf = new Configuration();
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
    conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

    // use local deploy mode and dummy dataspliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);
    conf.setInt(AngelConf.ANGEL_PSAGENT_CACHE_SYNC_TIMEINTERVAL_MS, 10000);

    // get a angel client
    angelClient = AngelClientFactory.get(conf);

    // add matrix
    MatrixContext mMatrix = new MatrixContext();
    mMatrix.setName("w1");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(100000);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(50000);
    mMatrix.setRowType(RowType.T_INT_DENSE);
    mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
    angelClient.addMatrix(mMatrix);

    MatrixContext mMatrix2 = new MatrixContext();
    mMatrix2.setName("w2");
    mMatrix2.setRowNum(1);
    mMatrix2.setColNum(100000);
    mMatrix2.setMaxRowNumInBlock(1);
    mMatrix2.setMaxColNumInBlock(50000);
    mMatrix2.setRowType(RowType.T_DOUBLE_DENSE);
    mMatrix2.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix2.set(MatrixConf.MATRIX_HOGWILD, "false");
    mMatrix2.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix2.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
    angelClient.addMatrix(mMatrix2);

    angelClient.startPSServer();
    angelClient.run();
    Thread.sleep(5000);
    group0Id = new WorkerGroupId(0);
    worker0Id = new WorkerId(group0Id, 0);
    worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
    task0Id = new TaskId(0);
    task1Id = new TaskId(1);
    psId = new ParameterServerId(0);
    psAttempt0Id = new PSAttemptId(psId, 0);
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：PSManagerTest.java    
@Before
public void setup() throws Exception {
  try {
    // set basic configuration keys
    Configuration conf = new Configuration();
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
    conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

    // use local deploy mode and dummy dataspliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);
    conf.setInt(AngelConf.ANGEL_PS_BACKUP_INTERVAL_MS, 5000);

    // get a angel client
    angelClient = AngelClientFactory.get(conf);

    // add matrix
    MatrixContext mMatrix = new MatrixContext();
    mMatrix.setName("w1");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(100000);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(50000);
    mMatrix.setRowType(RowType.T_INT_DENSE);
    mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
    angelClient.addMatrix(mMatrix);

    MatrixContext mMatrix2 = new MatrixContext();
    mMatrix2.setName("w2");
    mMatrix2.setRowNum(1);
    mMatrix2.setColNum(100000);
    mMatrix2.setMaxRowNumInBlock(1);
    mMatrix2.setMaxColNumInBlock(50000);
    mMatrix2.setRowType(RowType.T_DOUBLE_DENSE);
    mMatrix2.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix2.set(MatrixConf.MATRIX_HOGWILD, "false");
    mMatrix2.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix2.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
    angelClient.addMatrix(mMatrix2);

    angelClient.startPSServer();
    angelClient.run();
    Thread.sleep(5000);
    group0Id = new WorkerGroupId(0);
    worker0Id = new WorkerId(group0Id, 0);
    worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
    task0Id = new TaskId(0);
    task1Id = new TaskId(1);
    psId = new ParameterServerId(0);
    psAttempt0Id = new PSAttemptId(psId, 0);
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：AppTest.java    
@Before
public void setup() throws Exception {
  try{
    // set basic configuration keys
    Configuration conf = new Configuration();
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
    //conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

    // use local deploy mode and dummy dataspliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);

    // get a angel client
    angelClient = AngelClientFactory.get(conf);

    // add matrix
    MatrixContext mMatrix = new MatrixContext();
    mMatrix.setName("w1");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(100000);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(50000);
    mMatrix.setRowType(RowType.T_INT_DENSE);
    mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
    angelClient.addMatrix(mMatrix);

    angelClient.startPSServer();
    angelClient.runTask(DummyTask.class);
    Thread.sleep(5000);
    task0Id = new TaskId(0);
    task1Id = new TaskId(1);
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：MasterRecoverTest.java    
@Before
public void setup() throws Exception {
  try{
    // set basic configuration keys
    Configuration conf = new Configuration();
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
    conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

    // use local deploy mode and dummy dataspliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);

    // get a angel client
    angelClient = AngelClientFactory.get(conf);

    // add matrix
    MatrixContext mMatrix = new MatrixContext();
    mMatrix.setName("w1");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(100000);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(50000);
    mMatrix.setRowType(RowType.T_INT_DENSE);
    mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
    angelClient.addMatrix(mMatrix);

    MatrixContext mMatrix2 = new MatrixContext();
    mMatrix2.setName("w2");
    mMatrix2.setRowNum(1);
    mMatrix2.setColNum(100000);
    mMatrix2.setMaxRowNumInBlock(1);
    mMatrix2.setMaxColNumInBlock(50000);
    mMatrix2.setRowType(RowType.T_DOUBLE_DENSE);
    mMatrix2.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix2.set(MatrixConf.MATRIX_HOGWILD, "false");
    mMatrix2.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix2.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
    angelClient.addMatrix(mMatrix2);

    angelClient.startPSServer();
    angelClient.run();
    Thread.sleep(5000);
    group0Id = new WorkerGroupId(0);
    worker0Id = new WorkerId(group0Id, 0);
    worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
    task0Id = new TaskId(0);
    task1Id = new TaskId(1);
    psId = new ParameterServerId(0);
    psAttempt0Id = new PSAttemptId(psId, 0);
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}
项目：angel    文件：WorkerTest.java    
@BeforeClass
public static void setup() throws Exception {
  try {
    //set basic configuration keys
    conf = new Configuration();
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
    conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

    //use local deploy mode and dummy dataspliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);

    //get a angel client
    angelClient = AngelClientFactory.get(conf);

    //add matrix
    MatrixContext mMatrix = new MatrixContext();
    mMatrix.setName("w1");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(100000);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(50000);
    mMatrix.setRowType(RowType.T_INT_DENSE);
    mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
    angelClient.addMatrix(mMatrix);

    mMatrix = new MatrixContext();
    mMatrix.setName("w2");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(100000);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(50000);
    mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
    mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "false");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_DOUBLE");
    angelClient.addMatrix(mMatrix);

    angelClient.startPSServer();
    angelClient.run();
    Thread.sleep(5000);

    group0Id = new WorkerGroupId(0);
    worker0Id = new WorkerId(group0Id, 0);
    worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
    task0Id = new TaskId(0);
    task1Id = new TaskId(1);
  } catch (Exception x) {
    LOG.error("setup failed ", x);
    throw x;
  }
}