ml.dmlc.xgboost4j.java.XGBoostError Java Examples

public void setXGBooster(Booster booster) {
    this.booster = booster;
    try {
        Map<String, Integer> feaMap = booster.getFeatureScore(null);
        featureScores = new HashMap<>();
        for (Map.Entry<String, Integer> entry : feaMap.entrySet()) {
            String name = (String)indexSpace.getKeyForIndex(TreeKey.TREE.get(),
                    Integer.parseInt(entry.getKey().substring(1)));
            featureScores.put(name, entry.getValue());
        }
        logger.info("Number of non-zero importance features: {}", featureScores.size());
        logger.info("Feature importance: {}", Json.toJson(featureScores).toString());
    } catch (XGBoostError e) {
        throw new BadRequestException(e);
    }
}
 

private static DMatrix createSparseDMatrix() throws XGBoostError {
    /*
    11  12  13  14  0   0
    0   22  23  0   0   0
    0   0   33  34  35  36
    0   0   0   44  45  0
    0   0   0   0   0   56
    0   0   0   0   0   66
    */
    SparseDMatrixBuilder builder = new SparseDMatrixBuilder(1024);
    builder.nextRow(new String[] {"0:11", "1:12", "2:13", "3:14"});
    builder.nextRow(new String[] {"1:22", "2:23"});
    builder.nextRow(new String[] {"2:33", "3:34", "4:35", "5:36"});
    builder.nextRow(new String[] {"3:44", "4:45"});
    builder.nextRow(new String[] {"5:56"});
    builder.nextRow(new String[] {"5:66"});

    float[] labels = new float[6];
    return builder.buildMatrix(labels);
}
 

private static DMatrix createDenseDMatrix() throws XGBoostError {
    /*
    11  12  13  14  0   0
    0   22  23  0   0   0
    0   0   33  34  35  36
    0   0   0   44  45  0
    0   0   0   0   0   56
    0   0   0   0   0   66
    */
    DenseDMatrixBuilder builder = new DenseDMatrixBuilder(1024);
    builder.nextRow(new String[] {"0:11", "1:12", "2:13", "3:14"});
    builder.nextRow(new String[] {"1:22", "2:23"});
    builder.nextRow(new String[] {"2:33", "3:34", "4:35", "5:36"});
    builder.nextRow(new String[] {"3:44", "4:45"});
    builder.nextRow(new String[] {"5:56"});
    builder.nextRow(new String[] {"5:66"});

    float[] labels = new float[6];
    return builder.buildMatrix(labels);
}
 

@Test
public void testCreateFromCSREx() throws XGBoostError {
    // sparse matrix
    // 1 0 2 3 0
    // 4 0 2 3 5
    // 3 1 2 5 0
    DenseDMatrixBuilder builder = new DenseDMatrixBuilder(1024);
    builder.nextRow(new float[] {1, 0, 2, 3, 0});
    builder.nextRow(new float[] {4, 0, 2, 3, 5});
    builder.nextRow(new float[] {3, 1, 2, 5, 0});
    float[] label1 = new float[] {1, 0, 1};
    DMatrix dmat1 = builder.buildMatrix(label1);

    Assert.assertEquals(3, dmat1.rowNum());
    float[] label2 = dmat1.getLabel();
    Assert.assertArrayEquals(label1, label2, 0.f);
}
 

@Nonnull
private static Booster train(@Nonnull final DMatrix dtrain, @Nonnegative final int round,
        @Nonnull final Map<String, Object> params, @Nullable final Reporter reporter)
        throws NoSuchMethodException, IllegalAccessException, InvocationTargetException,
        InstantiationException, XGBoostError {
    final Counters.Counter iterCounter = (reporter == null) ? null
            : reporter.getCounter("hivemall.XGBoostTrainUDTF$Counter", "iteration");

    final Booster booster = XGBoostUtils.createBooster(dtrain, params);
    for (int iter = 0; iter < round; iter++) {
        reportProgress(reporter);
        setCounterValue(iterCounter, iter + 1);

        booster.update(dtrain, iter);
    }
    return booster;
}
 

@Override
public DMatrix buildMatrix(@Nonnull float[] labels) throws XGBoostError {
    final int numRows = rows.size();
    if (labels.length != numRows) {
        throw new XGBoostError(
            String.format("labels.length does not match to nrows. labels.length=%d, nrows=%d",
                labels.length, numRows));
    }

    final float[] data = new float[numRows * maxNumColumns];
    Arrays.fill(data, Float.NaN);
    for (int i = 0; i < numRows; i++) {
        final float[] row = rows.get(i);
        final int rowPtr = i * maxNumColumns;
        for (int j = 0; j < row.length; j++) {
            int ij = rowPtr + j;
            data[ij] = row[j];
        }
    }

    DMatrix matrix = new DMatrix(data, numRows, maxNumColumns, Float.NaN);
    matrix.setLabel(labels);
    return matrix;
}
 

public static DMatrix wekaInstancesToDMatrix(Instances insts) throws XGBoostError {
    int numRows = insts.numInstances();
    int numCols = insts.numAttributes()-1;

    float[] data = new float[numRows*numCols];
    float[] labels = new float[numRows];

    int ind = 0;
    for (int i = 0; i < numRows; i++) {
        for (int j = 0; j < numCols; j++)
            data[ind++] = (float) insts.instance(i).value(j);
        labels[i] = (float) insts.instance(i).classValue();
    }

    DMatrix dmat = new DMatrix(data, numRows, numCols);
    dmat.setLabel(labels);
    return dmat;
}
 

public static Async<Booster> asyncModel(final String modelFile,
		final int nthread) {
	// load xgboost model
	final Async<Booster> modelAsync = new Async<Booster>(() -> {
		try {
			Booster bst = XGBoost.loadModel(modelFile);
			if (nthread > 0) {
				bst.setParam("nthread", nthread);
			}
			return bst;
		} catch (XGBoostError e) {
			e.printStackTrace();
			return null;
		}
	}, Booster::dispose);
	return modelAsync;
}
 

@Nonnull
public DMatrix buildMatrix(@Nonnull float[] labels) throws XGBoostError {
    DMatrix matrix = new DMatrix(rowPointers.toArray(true), columnIndices.toArray(true),
        values.toArray(true), DMatrix.SparseType.CSR, maxNumColumns);
    matrix.setLabel(labels);
    return matrix;
}
 

public void saveModel(String modelFile) {
    try {
        this.booster.saveModel(modelFile);
    } catch (XGBoostError e) {
        throw new BadRequestException(e);
    }
}
 

public void learn(PredictiveModel model, LearningData learningData, LearningData validData) {
    try {
        DMatrix dtrain = new DMatrix(new XGBoostIterator(learningData), null);
        Map<String, DMatrix> watches = new HashMap<>();
        if (validData != null) {
            watches.put("Validation", new DMatrix(new XGBoostIterator(validData), null));
        }
        Booster booster = XGBoost.train(dtrain, params, round, watches, null, null);
        XGBoostModel boostModel = (XGBoostModel) model;
        boostModel.setXGBooster(booster);
    } catch (XGBoostError e) {
        throw new BadRequestException(e);
    }
}
 

private void predictAndFlush(@Nonnull final Booster model,
        @Nonnull final List<LabeledPointWithRowId> rowBatch) throws HiveException {
    DMatrix testData = null;
    final float[][] predicted;
    try {
        testData = XGBoostUtils.createDMatrix(rowBatch);
        predicted = model.predict(testData);
    } catch (XGBoostError e) {
        throw new HiveException("Exception caused at prediction", e);
    } finally {
        XGBoostUtils.close(testData);
    }
    forwardPredicted(rowBatch, predicted);
    rowBatch.clear();
}
 

@Nonnull
public static Booster createBooster(@Nonnull DMatrix matrix,
        @Nonnull Map<String, Object> params) throws NoSuchMethodException, XGBoostError,
        IllegalAccessException, InvocationTargetException, InstantiationException {
    Class<?>[] args = {Map.class, DMatrix[].class};
    Constructor<Booster> ctor = Booster.class.getDeclaredConstructor(args);
    ctor.setAccessible(true);
    return ctor.newInstance(new Object[] {params, new DMatrix[] {matrix}});
}
 

@Nonnull
public static DMatrix createDMatrix(@Nonnull final List<LabeledPointWithRowId> data)
        throws XGBoostError {
    final List<LabeledPoint> points = new ArrayList<>(data.size());
    for (LabeledPointWithRowId d : data) {
        points.add(d);
    }
    return new DMatrix(points.iterator(), "");
}
 

@GetMapping("/")
@ResponseBody
public ResponseWrapper handle(@RequestParam(name="text", required=false, defaultValue="我买了十斤水果") String value) throws XGBoostError {
    SmoothNLPResult result;

    result = SmoothNLP.process(value);
    return new ResponseWrapper(result);

}
 

public static DMatrix toDMatrix(final MLSparseMatrix matrix)
		throws XGBoostError {

	final int nnz = (int) matrix.getNNZ();
	final int nRows = matrix.getNRows();
	final int nCols = matrix.getNCols();

	long[] rowIndex = new long[nRows + 1];
	int[] indexesFlat = new int[nnz];
	float[] valuesFlat = new float[nnz];

	int cur = 0;
	for (int i = 0; i < nRows; i++) {
		MLSparseVector row = matrix.getRow(i);
		if (row == null) {
			rowIndex[i] = cur;
			continue;
		}
		int[] indexes = row.getIndexes();
		int rowNNZ = indexes.length;
		if (rowNNZ == 0) {
			rowIndex[i] = cur;
			continue;
		}
		float[] values = row.getValues();
		rowIndex[i] = cur;

		for (int j = 0; j < rowNNZ; j++, cur++) {
			indexesFlat[cur] = indexes[j];
			valuesFlat[cur] = values[j];
		}
	}
	rowIndex[nRows] = cur;
	return new DMatrix(rowIndex, indexesFlat, valuesFlat,
			DMatrix.SparseType.CSR, nCols);
}
 

public static int[] getFeatureImportance(final Booster model,
		final String[] featNames) throws XGBoostError {

	int[] importances = new int[featNames.length];
	// NOTE: not used feature are dropped here
	Map<String, Integer> importanceMap = model.getFeatureScore(null);

	for (Map.Entry<String, Integer> entry : importanceMap.entrySet()) {
		// get index from f0, f1 feature name output from xgboost
		int index = Integer.parseInt(entry.getKey().substring(1));
		importances[index] = entry.getValue();
	}

	return importances;
}
 

/**
 * Note: Please use Models from zoltar-models module.
 *
 * <p>Returns a XGBoost model given a URI to the serialized model file.
 */
public static XGBoostModel create(final Model.Id id, final URI modelUri) throws IOException {
  try {
    GompLoader.start();
    final InputStream is = Files.newInputStream(FileSystemExtras.path(modelUri));
    return new AutoValue_XGBoostModel(id, XGBoost.loadModel(is));
  } catch (final XGBoostError xgBoostError) {
    throw new IOException(xgBoostError);
  }
}
 

public static synchronized SmoothNLPResult process(String inputText) throws XGBoostError {

        SmoothNLPResult res = new SmoothNLPResult();

        long start = System.currentTimeMillis();
        List<SToken> sTokensPOS = POSTAG_PIPELINE.process(inputText);
        res.tokens = sTokensPOS;
        long end = System.currentTimeMillis();
        System.out.println();
        System.out.print("token size: "+res.tokens.size()+"; ");

        System.out.print("segment+postag time: ");
        System.out.print(end-start+" | ");

        start = System.currentTimeMillis();
        res.entities = NER_PIPELINE.process(res.tokens);
        end = System.currentTimeMillis();
        System.out.print("ner time: ");
        System.out.print(end-start+" | ");

        start = System.currentTimeMillis();
        DependencyRelationship[] dependencyRelationships=DEPENDENCY_PIPELINE.parse(res.tokens);
        res.dependencyRelationships = dependencyRelationships;
        end = System.currentTimeMillis();
        System.out.print("dependency time: ");
        System.out.print(end-start+" | ");
        return res;
    }
 

public static void trainXgbModel(String trainFile, String devFile, String modelAddr, int nround, int negSampleRate, int earlyStop, int nthreads) throws IOException{
    final DMatrix trainMatrix = readCoNLL2DMatrix(trainFile,negSampleRate);
    final DMatrix devMatrix = readCoNLL2DMatrix(devFile,negSampleRate);
    try{
        Map<String, Object> params = new HashMap<String, Object>() {
            {
                put("nthread", nthreads);
                put("max_depth", 16);
                put("silent", 0);
                put("objective", "binary:logistic");
                put("colsample_bytree",0.95);
                put("colsample_bylevel",0.95);
                put("eta",0.2);
                put("subsample",0.95);
                put("lambda",0.2);

                put("min_child_weight",5);
                put("scale_pos_weight",negSampleRate);

                // other parameters
                // "objective" -> "multi:softmax", "num_class" -> "6"

                put("eval_metric", "logloss");
                put("tree_method","approx");
            }
        };
        Map<String, DMatrix> watches = new HashMap<String, DMatrix>() {
            {
                put("train", trainMatrix);
                put("dev",devMatrix);
            }
        };
        Booster booster = XGBoost.train(trainMatrix, params, nround, watches, null, null,null,earlyStop);
        OutputStream outstream = SmoothNLP.IOAdaptor.create(modelAddr);
        booster.saveModel(outstream);
    }catch(XGBoostError e){
        System.out.println(e);
    }
}
 

public static void main(String[] args) throws XGBoostError{
        MaxEdgeScoreDependencyParser dparser = new MaxEdgeScoreDependencyParser();

//        long start = System.currentTimeMillis();
//        dparser.parse("邯郸市通达机械制造有限公司拥有固定资产1200万元，现有职工280名，其中专业技术人员80名，高级工程师两名，年生产能力10000吨，产值8000万元");
//        long end = System.currentTimeMillis();
//        System.out.print("dependency time: ");
//        System.out.println(end-start);
//
//        start = System.currentTimeMillis();
//        dparser.parse("邯郸市通达机械制造有限公司拥有固定资产1200万元，现有职工280名，其中专业技术人员80名，高级工程师两名，年生产能力10000吨，产值8000万元");
//        end = System.currentTimeMillis();
//        System.out.print("dependency time: ");
//        System.out.println(end-start);
        String text = "在面对用户的搜索产品不断丰富的同时，百度还创新性地推出了基于搜索的营销推广服务，并成为最受企业青睐的互联网营销推广平台。";
//        String text = "呼吸道飞沫传染是新型冠状病毒的主要传播途径";
        long start = System.currentTimeMillis();
        for (DependencyRelationship e : dparser.parse(text)) {
            System.out.println(e);
        }
        long end = System.currentTimeMillis();
        System.out.print("dependency time: ");
        System.out.println(end-start);
//        System.out.println(UtilFns.toJson(dparser.parse("阿里巴巴是以曾担任英语教师的马云为首的18人于1999年在浙江杭州创立的公司 ")));
//        System.out.println(UtilFns.toJson(dparser.parse("邯郸市通达机械制造有限公司拥有固定资产1200万元，现有职工280名，其中专业技术人员80名，高级工程师两名，年生产能力10000吨，产值8000万元")));
////        System.out.println(UtilFns.toJson(dparser.parse("中共中央政治局召开会议,分析研究2019年经济工作")));
//        System.out.println(UtilFns.toJson(dparser.parse("阿里与腾讯达成合作协议")));
//        System.out.println(UtilFns.toJson(dparser.parse("阿里巴巴在英属开曼群岛注册成立")));
//        System.out.println(UtilFns.toJson(dparser.parse("在面对用户的搜索产品不断丰富的同时，百度还创新性地推出了基于搜索的营销推广服务，并成为最受企业青睐的互联网营销推广平台。")));
//        System.out.println(UtilFns.toJson(dparser.parse("湖北华一专用汽车有限公司是随州专用汽车制造销售公司")));
//        System.out.println(UtilFns.toJson(dparser.parse("公开资料显示，香雪制药2016年通过股权受让的方式取得湖北天济55%股权。")));
//        System.out.println(UtilFns.toJson(dparser.parse("星晖新能源智能汽车生产基地是省重点发展项目之一，总投资超过200亿元，于2018年1月在黄冈产业园正式开工。")));

    }
 

public DependencyRelationship[] parse(List<SToken> stokens) throws XGBoostError{
    CoNLLDependencyGraph cgraph = new CoNLLDependencyGraph(stokens);
    // build ftrs
    Float[][] pairFtrs = cgraph.buildAllFtrs();
    float[] flattenPairFtrs = UtilFns.flatten2dFloatArray(pairFtrs);


    int numRecords = pairFtrs.length;
    int numFtrs = pairFtrs[0].length;
    DMatrix dmatrix = new DMatrix(flattenPairFtrs,numRecords,numFtrs);

    float[][] predictScores = this.edgeScoreModel.predict(dmatrix,false,SmoothNLP.XGBoost_DP_Edge_Model_Predict_Tree_Limit);  // 调节treeLimit , 优化时间

    float[] predictScoresFlatten = UtilFns.flatten2dFloatArray(predictScores);
    float[][] edgeScores = new float[cgraph.size()][cgraph.size()];
    for (int i =0; i<cgraph.size(); i++){
        for (int j = 0; j<cgraph.size(); j++){
            if (i!=j){  // 过滤一个token 自己依赖自己的情况
                // todo: 待评估
                edgeScores[i][j] = predictScoresFlatten[i*cgraph.size()+j];
            }
        }
    }
    cgraph.setEdgeScores(edgeScores);

    return cgraph.parseDependencyRelationships(this.edgeTagModel);
}
 

@Nonnull
public abstract DMatrix buildMatrix(@Nonnull float[] labels) throws XGBoostError;
 

@Nonnull
private static Booster train(@Nonnull final DMatrix dtrain, @Nonnull final DMatrix dtest,
        @Nonnegative final int round, @Nonnegative final int earlyStoppingRounds,
        @Nonnull final Map<String, Object> params, @Nullable final Reporter reporter)
        throws NoSuchMethodException, IllegalAccessException, InvocationTargetException,
        InstantiationException, XGBoostError {
    final Counters.Counter iterCounter = (reporter == null) ? null
            : reporter.getCounter("hivemall.XGBoostTrainUDTF$Counter", "iteration");

    final Booster booster = XGBoostUtils.createBooster(dtrain, params);

    final boolean maximizeEvaluationMetrics =
            OptionUtils.getBoolean(params, "maximize_evaluation_metrics");
    float bestScore = maximizeEvaluationMetrics ? -Float.MAX_VALUE : Float.MAX_VALUE;
    int bestIteration = 0;

    final float[] metricsOut = new float[1];
    for (int iter = 0; iter < round; iter++) {
        reportProgress(reporter);
        setCounterValue(iterCounter, iter + 1);

        booster.update(dtrain, iter);

        String evalInfo =
                booster.evalSet(new DMatrix[] {dtest}, new String[] {"test"}, iter, metricsOut);
        logger.info(evalInfo);

        final float score = metricsOut[0];
        if (maximizeEvaluationMetrics) {
            // Update best score if the current score is better (no update when equal)
            if (score > bestScore) {
                bestScore = score;
                bestIteration = iter;
            }
        } else {
            if (score < bestScore) {
                bestScore = score;
                bestIteration = iter;
            }
        }

        if (shouldEarlyStop(earlyStoppingRounds, iter, bestIteration)) {
            logger.info(
                String.format("early stopping after %d rounds away from the best iteration",
                    earlyStoppingRounds));
            break;
        }
    }

    return booster;
}
 

@Test
public void testDenseMatrix() throws XGBoostError {
    DMatrix matrix = createDenseDMatrix();
    Assert.assertEquals(6, matrix.rowNum());
    matrix.dispose();
}
 

@Test
public void testSparseMatrix() throws XGBoostError {
    DMatrix matrix = createSparseDMatrix();
    Assert.assertEquals(6, matrix.rowNum());
    matrix.dispose();
}
 

public DependencyRelationship[] parse(String input) throws XGBoostError{
    List<SToken> stokens = SmoothNLP.POSTAG_PIPELINE.process(input);
    return parse(stokens);
}
 

public static void trainXgbModel(String trainFile, String devFile, String modelAddr, int nround, int earlyStop,int nthreads ) throws IOException{
    final DMatrix trainMatrix = readCoNLL2DMatrix(trainFile);
    final DMatrix devMatrix = readCoNLL2DMatrix(devFile);
    try{
        Map<String, Object> params = new HashMap<String, Object>() {
            {
                put("nthread", nthreads);
                put("max_depth", 12);
                put("silent", 0);
                put("objective", "multi:softprob");
                put("colsample_bytree",0.90);
                put("colsample_bylevel",0.90);
                put("eta",0.2);
                put("subsample",0.95);
                put("lambda",1.0);

                // tree methods for regulation
                put("min_child_weight",5);
                put("max_leaves",128);

                // other parameters
                // "objective" -> "multi:softmax", "num_class" -> "6"

                put("eval_metric", "merror");
                put("tree_method","approx");
                put("num_class",tag2float.size());

                put("min_child_weight",5);
            }
        };
        Map<String, DMatrix> watches = new HashMap<String, DMatrix>() {
            {
                put("train", trainMatrix);
                put("dev",devMatrix);
            }
        };
        Booster booster = XGBoost.train(trainMatrix, params, nround, watches, null, null,null,earlyStop);
        OutputStream outstream = SmoothNLP.IOAdaptor.create(modelAddr);
        booster.saveModel(outstream);



    }catch(XGBoostError e){
        System.out.println(e);
    }
}
 

public DependencyRelationship[] parse(String input) throws XGBoostError {
    List<SToken> tokens = SmoothNLP.POSTAG_PIPELINE.process(input);
    return this.parse(tokens);
}
 

public DependencyRelationship[] parse(List<SToken> stokens) throws XGBoostError;