org.deeplearning4j.nn.layers.objdetect.DetectedObject Java Examples

private static void cropImageWithYOLOBoundingBox(ComputationGraph yolo,
                                                 Speed selectedSpeed, File file) throws Exception {
    if (file.isDirectory()) {
        return;
    }
    BufferedImage bufferedImage = ImageIO.read(file);
    INDArray features = LOADER.asMatrix(bufferedImage);
    opencv_core.Mat mat = LOADER.asMat(features);
    PRE_PROCESSOR.transform(features);
    INDArray results = yolo.outputSingle(features);
    Yolo2OutputLayer outputLayer = (Yolo2OutputLayer) yolo.getOutputLayer(0);
    List<DetectedObject> predictedObjects = outputLayer.getPredictedObjects(results, 0.5);
    YoloUtils.nms(predictedObjects, 0.5);
    Optional<DetectedObject> max = predictedObjects.stream()
            .max((o1, o2) -> ((Double) o1.getConfidence()).compareTo(o2.getConfidence()));
    createCroppedImage(mat, selectedSpeed, max.get(), file);
}
 

private void drawBoxes(List<DetectedObject> predictedObjects) {
	ctx.setLineWidth(3);
	int w = yoloModel.getInputWidth();
	int h = yoloModel.getInputHeight();
	int gridW = yoloModel.getGridW();
	int gridH = yoloModel.getGridH();
	ctx.setTextAlign(TextAlignment.CENTER);
	for (DetectedObject obj : predictedObjects) {
		String cl = yoloModel.getModelClasses()[obj.getPredictedClass()];
		double[] xy1 = obj.getTopLeftXY();
		double[] xy2 = obj.getBottomRightXY();
		int x1 = (int) Math.round(w * xy1[0] / gridW);
		int y1 = (int) Math.round(h * xy1[1] / gridH);
		int x2 = (int) Math.round(w * xy2[0] / gridW);
		int y2 = (int) Math.round(h * xy2[1] / gridH);
		int rectW = x2 - x1;
		int rectH = y2 - y1;
		ctx.setStroke(colors.get(cl));
		ctx.strokeRect(x1, y1, rectW, rectH);
		ctx.strokeText(cl, x1 + (rectW / 2), y1 - 2);
		ctx.setFill(Color.WHITE);
		ctx.fillText(cl, x1 + (rectW / 2), y1 - 2);
	}
}
 

private DetectedObjectsBatch[] getPredictedObjects(INDArray[] outputs, double threshold, String[] outputNames, int originalHeight, int originalWidth) {
    // assuming "standard" output from TensorFlow using a "normal" YOLOv2 model
    //INDArray permuted = outputs[0].permute(0, 3, 1, 2);
    INDArray permuted = outputs[0];
    INDArray activated = YoloUtils.activate(boundingBoxPriors, permuted);

    List<DetectedObject> predictedObjects1 = YoloUtils.getPredictedObjects(boundingBoxPriors, activated, threshold, 0.4);


    DetectedObjectsBatch[] detectedObjects = new DetectedObjectsBatch[predictedObjects1.size()];

    int n = numLabels; // an arbitrary number of classes returned per object
    for (int i = 0; i < detectedObjects.length; i++) {
        DetectedObject detectedObject = predictedObjects1.get(i);
        long x = Math.round(originalWidth * predictedObjects1.get(i).getCenterX() / gridWidth);
        long y = Math.round(originalHeight * predictedObjects1.get(i).getCenterY() / gridHeight);
        long w = Math.round(originalWidth * predictedObjects1.get(i).getWidth() / gridWidth);
        long h = Math.round(originalHeight * predictedObjects1.get(i).getHeight() / gridHeight);

        detectedObjects[i] = new DetectedObjectsBatch();
        detectedObjects[i].setCenterX(x);
        detectedObjects[i].setCenterY(y);
        detectedObjects[i].setWidth(w);
        detectedObjects[i].setHeight(h);
        detectedObjects[i].setPredictedClasses(new String[]{labels.getLabel(detectedObject.getPredictedClass())});
        detectedObjects[i].setPredictedClassNumbers(new int[]{detectedObject.getPredictedClass()});
        detectedObjects[i].setConfidences(new float[]{detectedObject.getClassPredictions().getFloat(detectedObject.getPredictedClass())});

    }

    return detectedObjects;


}
 

public INDArray getEmbeddings(opencv_core.Mat file, DetectedObject obj, Speed selectedSpeed) throws Exception {
    BufferedImage croppedImage = ImageUtils.cropImageWithYolo(selectedSpeed,
            file, obj, false);
    INDArray croppedContent = LOADER.asMatrix(croppedImage);
    IMAGE_PRE_PROCESSOR.transform(croppedContent);

    Map<String, INDArray> stringINDArrayMap = getCifar10Transfer()
            .feedForward(croppedContent, false);
    INDArray indArray = stringINDArrayMap.get(CONTENT_LAYER_NAME);

    return indArray;
}
 

public MarkedObject(DetectedObject detectedObject, INDArray l2Norm, long created, opencv_core.Mat matFrame) {
    this.detectedObject = detectedObject;
    this.l2Norm = l2Norm;
    this.created = created;
    this.matFrame = matFrame;
    id = "" + ID.incrementAndGet();
}
 

public void predictBoundingBoxes(String windowName) throws Exception {
    long start = System.currentTimeMillis();
    Yolo2OutputLayer outputLayer = (Yolo2OutputLayer) modelsMap.get(windowName).getOutputLayer(0);
    Mat matFrame = stackMap.get(windowName).pop();
    INDArray indArray = prepareImage(matFrame);
    log.info("stack of frames size " + stackMap.get(windowName).size());
    if (indArray == null) {
        return;
    }

    INDArray results = modelsMap.get(windowName).outputSingle(indArray);
    if (results == null) {
        return;
    }
    List<DetectedObject> predictedObjects = outputLayer.getPredictedObjects(results, YOLO_DETECTION_THRESHOLD);
    YoloUtils.nms(predictedObjects, 0.5);
    List<MarkedObject> markedObjects = predictedObjects.stream()
            .filter(e -> groupMap.get(map.get(e.getPredictedClass())) != null)
            .map(e -> {
                try {
                    return new MarkedObject(e, trainCifar10Model.getEmbeddings(matFrame, e, selectedSpeed),
                            System.currentTimeMillis(), matFrame);
                } catch (Exception e1) {
                    e1.printStackTrace();
                }
                return null;
            }).collect(Collectors.toCollection(() -> new ConcurrentArrayList<>()));
    if (outputFrames) {
        for (MarkedObject markedObject : markedObjects) {
            ImageUtils.cropImageWithYolo(selectedSpeed, markedObject.getMatFrame(),
                    markedObject.getDetectedObject(), outputFrames);
        }
    }
    this.predictedObjects = markedObjects;
    log.info("stack of predictions size " + this.predictedObjects.size());
    log.info("Prediction time " + (System.currentTimeMillis() - start) / 1000d);
}
 

public void drawBoundingBoxesRectangles(Frame frame, Mat matFrame) {
    if (invalidData(frame, matFrame)) return;

    ArrayList<DetectedObject> detectedObjects = new ArrayList<>(predictedObjects);
    YoloUtils.nms(detectedObjects, 0.5);
    for (DetectedObject detectedObject : detectedObjects) {
        createBoundingBoxRectangle(matFrame, frame.imageWidth, frame.imageHeight, detectedObject);
    }

}
 

private void createBoundingBoxRectangle(Mat file, int w, int h, DetectedObject obj) {

        double[] xy1 = obj.getTopLeftXY();
        double[] xy2 = obj.getBottomRightXY();
        int predictedClass = obj.getPredictedClass();
        int x1 = (int) Math.round(w * xy1[0] / selectedSpeed.gridWidth);
        int y1 = (int) Math.round(h * xy1[1] / selectedSpeed.gridHeight);
        int x2 = (int) Math.round(w * xy2[0] / selectedSpeed.gridWidth);
        int y2 = (int) Math.round(h * xy2[1] / selectedSpeed.gridHeight);
        rectangle(file, new Point(x1, y1), new Point(x2, y2), Scalar.RED);
        putText(file, groupMap.get(map.get(predictedClass)), new Point(x1 + 2, y2 - 2), FONT_HERSHEY_DUPLEX, 1, Scalar.GREEN);
    }
 

public List<DetectedObject> run(File input, double threshold) {
	INDArray img;
	try {
		img = loadImage(input);
	} catch (IOException e) {
		throw new Error("Not able to load image from: " + input.getAbsolutePath(), e);
	}
	INDArray output = yoloModel.outputSingle(img);
	Yolo2OutputLayer outputLayer = (Yolo2OutputLayer) yoloModel.getOutputLayer(0);
	return outputLayer.getPredictedObjects(output, threshold);
}
 

private void update() {
	AppUtils.doBlockingAsyncWork(scene, () -> {
		ctx.clearRect(0, 0, yoloModel.getInputWidth(), yoloModel.getInputHeight());
		ctx.drawImage(currentImageFXImage, 0, 0);
		List<DetectedObject> detectedObjects = yoloModel.run(currentImage, sldThreshold.getValue());
		drawBoxes(detectedObjects);
	});
}
 

@Override
public List<DetectedObject> apply(Model model, INDArray[] inputs, INDArray[] masks, INDArray[] labelsMasks) {
    if (model instanceof ComputationGraph) {
        val blindLayer = ((ComputationGraph) model).getOutputLayer(outputLayerIndex);
        if (blindLayer instanceof Yolo2OutputLayer) {
            val output = ((ComputationGraph) model).output(false, inputs, masks, labelsMasks);
            return ((Yolo2OutputLayer) blindLayer).getPredictedObjects(output[outputIndex], detectionThreshold);
        } else {
            throw new ND4JIllegalStateException("Output layer with index [" + outputLayerIndex + "] is NOT Yolo2OutputLayer");
        }
    } else
        throw new ND4JIllegalStateException("Yolo2 model must be ComputationGraph");
}
 

public DetectedObject getDetectedObject() {
    return detectedObject;
}
 

private void renderJavaFXStyle(GraphicsContext ctx) throws Exception {

        INDArray boundingBoxPriors = Nd4j.create(YOLO2.DEFAULT_PRIOR_BOXES);

        ctx.setLineWidth(3);
        ctx.setTextAlign(TextAlignment.LEFT);

        long start = System.nanoTime();
        for (int i = 0; i < 1; i++) {
            INDArray permuted = networkGlobalOutput.permute(0, 3, 1, 2);
            INDArray activated = YoloUtils.activate(boundingBoxPriors, permuted);
            List<DetectedObject> predictedObjects = YoloUtils.getPredictedObjects(boundingBoxPriors, activated, 0.6, 0.4);
            
            //System.out.println( "width: " + imageWidth );
            //System.out.println( "height: " + imageHeight );

            for (DetectedObject o : predictedObjects) {
                ClassPrediction classPrediction = labels.decodePredictions(o.getClassPredictions(), 1).get(0).get(0);
                String label = classPrediction.getLabel();
                long x = Math.round(imageWidth  * o.getCenterX() / gridWidth);
                long y = Math.round(imageHeight * o.getCenterY() / gridHeight);
                long w = Math.round(imageWidth  * o.getWidth()   / gridWidth);
                long h = Math.round(imageHeight * o.getHeight()  / gridHeight);

                System.out.println("\"" + label + "\" at [" + x + "," + y + ";" + w + "," + h + "], score = "
                                + o.getConfidence() * classPrediction.getProbability());

                double[] xy1 = o.getTopLeftXY();
                double[] xy2 = o.getBottomRightXY();
                int x1 = (int) Math.round(imageWidth  * xy1[0] / gridWidth);
                int y1 = (int) Math.round(imageHeight * xy1[1] / gridHeight);
                int x2 = (int) Math.round(imageWidth  * xy2[0] / gridWidth);
                int y2 = (int) Math.round(imageHeight * xy2[1] / gridHeight);

                int rectW = x2 - x1;
                int rectH = y2 - y1;
                //System.out.printf("%s - %d, %d, %d, %d \n", label, x1, x2, y1, y2);
                //System.out.println( "color: " + colors.get(label) );
                ctx.setStroke(colors.get(label));
                ctx.strokeRect(x1, y1, rectW, rectH);
                
                int labelWidth = label.length() * 10;
                int labelHeight = 14;
                
                ctx.setFill( colors.get(label) );
                ctx.strokeRect(x1, y1-labelHeight, labelWidth, labelHeight);
                ctx.fillRect(x1, y1 - labelHeight, labelWidth, labelHeight);
                ctx.setFill( Color.WHITE );
                ctx.fillText(label, x1 + 3, y1 - 3 );

            }

        }

        globalEndTime = System.nanoTime();
        long end = System.nanoTime();
        System.out.println("Rendering code took: " + (end - start) / 1000000 + " ms");

        System.out.println("Overall Program Time: " + (globalEndTime - globalStartTime) / 1000000 + " ms");

    }
 

@Override
public List<DetectedObject> apply(INDArray... outputs) {
    throw new UnsupportedOperationException("Please use apply(Model, INDArray[], INDArray[]) signature");
}