Java Code Examples for org.datavec.api.conf.Configuration#getInt()

The following examples show how to use org.datavec.api.conf.Configuration#getInt() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TextVectorizer.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
@Override
public void initialize(Configuration conf) {
    tokenizerFactory = createTokenizerFactory(conf);
    minWordFrequency = conf.getInt(MIN_WORD_FREQUENCY, 5);
    if(conf.get(STOP_WORDS) != null)
        stopWords = conf.getStringCollection(STOP_WORDS);
    if (stopWords == null)
        stopWords = StopWords.getStopWords();

    String clazz = conf.get(VOCAB_CACHE, DefaultVocabCache.class.getName());
    try {
        Class<? extends VocabCache> tokenizerFactoryClazz = (Class<? extends VocabCache>) Class.forName(clazz);
        cache = tokenizerFactoryClazz.newInstance();
        cache.initialize(conf);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
Example 2
Source File: BaseImageRecordReader.java    From DataVec with Apache License 2.0 6 votes vote down vote up 
@Override
public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
    this.appendLabel = conf.getBoolean(APPEND_LABEL, appendLabel);
    this.labels = new ArrayList<>(conf.getStringCollection(LABELS));
    this.height = conf.getInt(HEIGHT, height);
    this.width = conf.getInt(WIDTH, width);
    this.channels = conf.getInt(CHANNELS, channels);
    this.cropImage = conf.getBoolean(CROP_IMAGE, cropImage);
    if ("imageio".equals(conf.get(IMAGE_LOADER))) {
        this.imageLoader = new ImageLoader(height, width, channels, cropImage);
    } else {
        this.imageLoader = new NativeImageLoader(height, width, channels, imageTransform);
    }
    this.conf = conf;
    initialize(split);
}
Example 3
Source File: TextVectorizer.java    From DataVec with Apache License 2.0 6 votes vote down vote up 
@Override
public void initialize(Configuration conf) {
    tokenizerFactory = createTokenizerFactory(conf);
    minWordFrequency = conf.getInt(MIN_WORD_FREQUENCY, 5);
    stopWords = conf.getStringCollection(STOP_WORDS);
    if (stopWords == null || stopWords.isEmpty())
        stopWords = StopWords.getStopWords();

    String clazz = conf.get(VOCAB_CACHE, DefaultVocabCache.class.getName());
    try {
        Class<? extends VocabCache> tokenizerFactoryClazz = (Class<? extends VocabCache>) Class.forName(clazz);
        cache = tokenizerFactoryClazz.newInstance();
        cache.initialize(conf);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
Example 4
Source File: BaseCodecRecordReader.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Override
public void setConf(Configuration conf) {
    super.setConf(conf);
    startFrame = conf.getInt(START_FRAME, 0);
    numFrames = conf.getInt(TOTAL_FRAMES, -1);
    rows = conf.getInt(ROWS, 28);
    cols = conf.getInt(COLUMNS, 28);
    framesPerSecond = conf.getFloat(TIME_SLICE, -1);
    videoLength = conf.getFloat(VIDEO_DURATION, -1);
    ravel = conf.getBoolean(RAVEL, false);
    totalFrames = conf.getInt(TOTAL_FRAMES, -1);
}
Example 5
Source File: SVMLightRecordWriter.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
/**
 * Set DataVec configuration
 *
 * @param conf
 */
@Override
public void setConf(Configuration conf) {
    super.setConf(conf);
    featureFirstColumn = conf.getInt(FEATURE_FIRST_COLUMN, 0);
    hasLabel = conf.getBoolean(HAS_LABELS, true);
    multilabel = conf.getBoolean(MULTILABEL, false);
    labelFirstColumn = conf.getInt(LABEL_FIRST_COLUMN, -1);
    labelLastColumn = conf.getInt(LABEL_LAST_COLUMN, -1);
    featureLastColumn = conf.getInt(FEATURE_LAST_COLUMN, labelFirstColumn > 0 ? labelFirstColumn-1 : -1);
    zeroBasedIndexing = conf.getBoolean(ZERO_BASED_INDEXING, false);
    zeroBasedLabelIndexing = conf.getBoolean(ZERO_BASED_LABEL_INDEXING, false);
}
Example 6
Source File: SVMLightRecordWriter.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
/**
 * Set DataVec configuration
 *
 * @param conf
 */
@Override
public void setConf(Configuration conf) {
    super.setConf(conf);
    featureFirstColumn = conf.getInt(FEATURE_FIRST_COLUMN, 0);
    hasLabel = conf.getBoolean(HAS_LABELS, true);
    multilabel = conf.getBoolean(MULTILABEL, false);
    labelFirstColumn = conf.getInt(LABEL_FIRST_COLUMN, -1);
    labelLastColumn = conf.getInt(LABEL_LAST_COLUMN, -1);
    featureLastColumn = conf.getInt(FEATURE_LAST_COLUMN, labelFirstColumn > 0 ? labelFirstColumn-1 : -1);
    zeroBasedIndexing = conf.getBoolean(ZERO_BASED_INDEXING, false);
    zeroBasedLabelIndexing = conf.getBoolean(ZERO_BASED_LABEL_INDEXING, false);
}
Example 7
Source File: NumberOfRecordsPartitioner.java    From DataVec with Apache License 2.0 4 votes vote down vote up 
@Override
public void init(Configuration configuration, InputSplit split) {
    init(split);
    this.recordsPerFile = configuration.getInt(RECORDS_PER_FILE_CONFIG,DEFAULT_RECORDS_PER_FILE);
}
Example 8
Source File: VasttextVocabCache.java    From scava with Eclipse Public License 2.0 4 votes vote down vote up 
@Override
public void initialize(Configuration conf) {
    minWordFrequency = conf.getInt(VasttextTextVectorizer.MIN_WORD_FREQUENCY, 5);
}
Example 9
Source File: CSVRecordReader.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Override
public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
    super.initialize(conf, split);
    this.skipNumLines = conf.getInt(SKIP_NUM_LINES, this.skipNumLines);
    this.csvParser = new SerializableCSVParser(conf.getChar(DELIMITER, DEFAULT_DELIMITER), conf.getChar(QUOTE, DEFAULT_QUOTE));
}
Example 10
Source File: CSVVariableSlidingWindowRecordReader.java    From DataVec with Apache License 2.0 4 votes vote down vote up 
@Override
public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
    super.initialize(conf, split);
    this.maxLinesPerSequence = conf.getInt(LINES_PER_SEQUENCE, maxLinesPerSequence);
}
Example 11
Source File: CSVRecordReader.java    From DataVec with Apache License 2.0 4 votes vote down vote up 
@Override
public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
    super.initialize(conf, split);
    this.skipNumLines = conf.getInt(SKIP_NUM_LINES, this.skipNumLines);
    this.csvParser = new SerializableCSVParser(conf.getChar(DELIMITER, DEFAULT_DELIMITER), conf.getChar(QUOTE, DEFAULT_QUOTE));
}
Example 12
Source File: RegexLineRecordReader.java    From DataVec with Apache License 2.0 4 votes vote down vote up 
@Override
public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
    super.initialize(conf, split);
    this.skipNumLines = conf.getInt(SKIP_NUM_LINES, this.skipNumLines);
}
Example 13
Source File: RegexSequenceRecordReader.java    From DataVec with Apache License 2.0 4 votes vote down vote up 
@Override
public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
    super.initialize(conf, split);
    this.skipNumLines = conf.getInt(SKIP_NUM_LINES, this.skipNumLines);
}
Example 14
Source File: CSVVariableSlidingWindowRecordReader.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Override
public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
    super.initialize(conf, split);
    this.maxLinesPerSequence = conf.getInt(LINES_PER_SEQUENCE, maxLinesPerSequence);
}
Example 15
Source File: CSVNLinesSequenceRecordReader.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Override
public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
    super.initialize(conf, split);
    this.nLinesPerSequence = conf.getInt(LINES_PER_SEQUENCE, nLinesPerSequence);
}
Example 16
Source File: ExcelRecordReader.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Override
public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
    super.initialize(conf, split);
    this.skipNumLines = conf.getInt(SKIP_NUM_LINES,0);
}
Example 17
Source File: NumberOfRecordsPartitioner.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Override
public void init(Configuration configuration, InputSplit split) {
    init(split);
    this.recordsPerFile = configuration.getInt(RECORDS_PER_FILE_CONFIG,DEFAULT_RECORDS_PER_FILE);
}
Example 18
Source File: DataIteratorConstructor.java    From scava with Eclipse Public License 2.0 4 votes vote down vote up 
private void buildDataIterator(Configuration configuration, FileDataSet fileDataSet, VasttextTextVectorizer vectorizer) throws IOException, InterruptedException
{
	Builder vasttextDataIteratorBuilder = new VasttextDataIterator.Builder(configuration.getInt(batchSize, 32));
	
	configuration.setBoolean(VasttextTextFileReader.LABELLED, fileDataSet.isLabelled());
	configuration.setBoolean(VasttextTextFileReader.MULTILABEL, fileDataSet.isMultiLabel());
	
	VasttextTextFileReader textFileReader = new VasttextTextFileReader();
	if(vectorizer!=null)
		textFileReader.setVastTextTextVectorizer(vectorizer);
	textFileReader.initialize(configuration, new FileSplit(fileDataSet.getTextFilePath().toFile()));
	
	multilabel=fileDataSet.isMultiLabel();
	textFeaturesSize=textFileReader.getTextFeaturesSize();
	labels=textFileReader.getLabels();
	
	if(labels!=null && storeVectorizer)
	{
		this.vectorizer=textFileReader.getVasttextTextVectorizer();
	}
	else
	{
		storeVectorizer=false;
		this.vectorizer=null;
	}
	
	vasttextDataIteratorBuilder.addReader("textReader", textFileReader);
	if(labels!=null)
		vasttextDataIteratorBuilder.addOutput("textReader", labels.size());
	vasttextDataIteratorBuilder.addInput("textReader");
	
	if(fileDataSet.getExtraFilePath()!=null)
	{
		configuration.setInt(VasttextExtraFileReader.FEATURES, fileDataSet.numericFeaturesSize());
		
		VasttextExtraFileReader extraFileReader = new VasttextExtraFileReader(textFileReader.getLinesDeleted());
		extraFileReader.initialize(configuration, new FileSplit(fileDataSet.getExtraFilePath().toFile()));
		
		numericFeaturesSize=fileDataSet.numericFeaturesSize();
		
		vasttextDataIteratorBuilder.addReader("extraReader", extraFileReader);
		vasttextDataIteratorBuilder.addInput("extraReader");
	}

	dataIterator = vasttextDataIteratorBuilder.build();
	
}
Example 19
Source File: VasttextExtraFileReader.java    From scava with Eclipse Public License 2.0 4 votes vote down vote up 
@Override
public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
    super.initialize(conf, split);
    numericFeaturesSize=conf.getInt(FEATURES, -1);
    checkNumericFeatures();
}
Example 20
Source File: VasttextDictionary.java    From scava with Eclipse Public License 2.0 4 votes vote down vote up 
public void initialize(Configuration conf) {
    minWordFrequency = conf.getInt(VasttextTextVectorizer.MIN_WORD_FREQUENCY, 5);
}