Java Code Examples for org.datavec.api.writable.Writable#toString()

The following examples show how to use org.datavec.api.writable.Writable#toString() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: WritablesToStringFunction.java From deeplearning4j with Apache License 2.0

6 votes

public static void append(List<Writable> c, StringBuilder sb, String delim, String quote) {
    boolean first = true;
    for (Writable w : c) {
        if (!first)
            sb.append(delim);
        String s = w.toString();
        boolean needQuotes = s.contains(delim);
        if (needQuotes && quote != null) {
            sb.append(quote);
            s = s.replace(quote, quote + quote);
        }
        sb.append(s);
        if (needQuotes && quote != null) {
            sb.append(quote);
        }
        first = false;
    }
}

Example 2

Source File: LongQualityAddFunction.java From DataVec with Apache License 2.0

6 votes

@Override
public LongQuality call(LongQuality v1, Writable writable) throws Exception {

    long valid = v1.getCountValid();
    long invalid = v1.getCountInvalid();
    long countMissing = v1.getCountMissing();
    long countTotal = v1.getCountTotal() + 1;
    long nonLong = v1.getCountNonLong();

    if (meta.isValid(writable))
        valid++;
    else if (writable instanceof NullWritable
                    || writable instanceof Text && (writable.toString() == null || writable.toString().isEmpty()))
        countMissing++;
    else
        invalid++;

    String str = writable.toString();
    try {
        Long.parseLong(str);
    } catch (NumberFormatException e) {
        nonLong++;
    }

    return new LongQuality(valid, invalid, countMissing, countTotal, nonLong);
}

Example 3

Source File: WritablesToStringFunction.java From DataVec with Apache License 2.0

6 votes

public static void append(List<Writable> c, StringBuilder sb, String delim, String quote) {
    boolean first = true;
    for (Writable w : c) {
        if (!first)
            sb.append(delim);
        String s = w.toString();
        boolean needQuotes = s.contains(delim);
        if (needQuotes && quote != null) {
            sb.append(quote);
            s = s.replace(quote, quote + quote);
        }
        sb.append(s);
        if (needQuotes && quote != null) {
            sb.append(quote);
        }
        first = false;
    }
}

Example 4

Source File: CategoricalQualityAddFunction.java From DataVec with Apache License 2.0

6 votes

@Override
public CategoricalQuality call(CategoricalQuality v1, Writable writable) throws Exception {

    long valid = v1.getCountValid();
    long invalid = v1.getCountInvalid();
    long countMissing = v1.getCountMissing();
    long countTotal = v1.getCountTotal() + 1;

    if (meta.isValid(writable))
        valid++;
    else if (writable instanceof NullWritable
                    || writable instanceof Text && (writable.toString() == null || writable.toString().isEmpty()))
        countMissing++;
    else
        invalid++;

    return new CategoricalQuality(valid, invalid, countMissing, countTotal);
}

Example 5

Source File: StringQualityAddFunction.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public StringQuality apply(StringQuality v1, Writable writable) {
    long valid = v1.getCountValid();
    long invalid = v1.getCountInvalid();
    long countMissing = v1.getCountMissing();
    long countTotal = v1.getCountTotal() + 1;
    long empty = v1.getCountEmptyString();
    long alphabetic = v1.getCountAlphabetic();
    long numerical = v1.getCountNumerical();
    long word = v1.getCountWordCharacter();
    long whitespaceOnly = v1.getCountWhitespace();
    HyperLogLogPlus hll = v1.getHll();

    String str = writable.toString();

    if (writable instanceof NullWritable)
        countMissing++;
    else if (meta.isValid(writable))
        valid++;
    else
        invalid++;

    if (str == null || str.isEmpty()) {
        empty++;
    } else {
        if (str.matches("[a-zA-Z]"))
            alphabetic++;
        if (str.matches("\\d+"))
            numerical++;
        if (str.matches("\\w+"))
            word++;
        if (str.matches("\\s+"))
            whitespaceOnly++;
    }

    hll.offer(str);
    return new StringQuality(valid, invalid, countMissing, countTotal, empty, alphabetic, numerical, word,
                    whitespaceOnly, hll);
}

Example 6

Source File: FilterWritablesBySchemaFunction.java From DataVec with Apache License 2.0

5 votes

@Override
public Boolean call(Writable v1) throws Exception {
    boolean valid = meta.isValid(v1);
    if (excludeMissing && (v1 instanceof NullWritable
                    || v1 instanceof Text && (v1.toString() == null || v1.toString().isEmpty())))
        return false; //Remove (spark)
    if (keepValid)
        return valid; //Spark: return true to keep
    else
        return !valid;
}

Example 7

Source File: FilterWritablesBySchemaFunction.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public Boolean apply(Writable v1) {
    boolean valid = meta.isValid(v1);
    if (excludeMissing && (v1 instanceof NullWritable
                    || v1 instanceof Text && (v1.toString() == null || v1.toString().isEmpty())))
        return false; //Remove
    if (keepValid)
        return valid; //return true to keep
    else
        return !valid;
}

Example 8

Source File: StringMetaData.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public boolean isValid(Writable writable) {
    String str = writable.toString();
    int len = str.length();
    if (minLength != null && len < minLength)
        return false;
    if (maxLength != null && len > maxLength)
        return false;

    return regex == null || str.matches(regex);
}

Example 9

Source File: StringQualityAddFunction.java From DataVec with Apache License 2.0

5 votes

@Override
public StringQuality call(StringQuality v1, Writable writable) throws Exception {
    long valid = v1.getCountValid();
    long invalid = v1.getCountInvalid();
    long countMissing = v1.getCountMissing();
    long countTotal = v1.getCountTotal() + 1;
    long empty = v1.getCountEmptyString();
    long alphabetic = v1.getCountAlphabetic();
    long numerical = v1.getCountNumerical();
    long word = v1.getCountWordCharacter();
    long whitespaceOnly = v1.getCountWhitespace();
    HyperLogLogPlus hll = v1.getHll();

    String str = writable.toString();

    if (writable instanceof NullWritable)
        countMissing++;
    else if (meta.isValid(writable))
        valid++;
    else
        invalid++;

    if (str == null || str.isEmpty()) {
        empty++;
    } else {
        if (str.matches("[a-zA-Z]"))
            alphabetic++;
        if (str.matches("\\d+"))
            numerical++;
        if (str.matches("\\w+"))
            word++;
        if (str.matches("\\s+"))
            whitespaceOnly++;
    }

    hll.offer(str);
    return new StringQuality(valid, invalid, countMissing, countTotal, empty, alphabetic, numerical, word,
                    whitespaceOnly, hll);
}

Example 10

Source File: RealQualityAddFunction.java From DataVec with Apache License 2.0

5 votes

@Override
public DoubleQuality call(DoubleQuality v1, Writable writable) throws Exception {

    long valid = v1.getCountValid();
    long invalid = v1.getCountInvalid();
    long countMissing = v1.getCountMissing();
    long countTotal = v1.getCountTotal() + 1;
    long nonReal = v1.getCountNonReal();
    long nan = v1.getCountNaN();
    long infinite = v1.getCountInfinite();

    if (meta.isValid(writable))
        valid++;
    else if (writable instanceof NullWritable
                    || writable instanceof Text && (writable.toString() == null || writable.toString().isEmpty()))
        countMissing++;
    else
        invalid++;

    String str = writable.toString();
    double d;
    try {
        d = Double.parseDouble(str);
        if (Double.isNaN(d))
            nan++;
        if (Double.isInfinite(d))
            infinite++;
    } catch (NumberFormatException e) {
        nonReal++;
    }

    return new DoubleQuality(valid, invalid, countMissing, countTotal, nonReal, nan, infinite);
}

Example 11

Source File: CategoricalAnalysisCounter.java From DataVec with Apache License 2.0

5 votes

@Override
public CategoricalAnalysisCounter add(Writable writable) {
    String value = writable.toString();

    long newCount = 0;
    if (counts.containsKey(value)) {
        newCount = counts.get(value);
    }
    newCount++;
    counts.put(value, newCount);

    countTotal++;

    return this;
}

Example 12

Source File: ReplaceEmptyStringTransform.java From DataVec with Apache License 2.0

5 votes

@Override
public Text map(Writable writable) {
    String s = writable.toString();
    if (s == null || s.isEmpty())
        return new Text(value);
    else if (writable instanceof Text)
        return (Text) writable;
    else
        return new Text(writable.toString());
}

Example 13

Source File: MapAllStringsExceptListTransform.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public Text map(Writable writable) {
    String str = writable.toString();
    if (exceptions.contains(str)) {
        return new Text(str);
    } else {
        return new Text(newValue);
    }
}

Example 14

Source File: CategoricalToOneHotTransform.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public List<Writable> map(List<Writable> writables) {
    if (writables.size() != inputSchema.numColumns()) {
        throw new IllegalStateException("Cannot execute transform: input writables list length (" + writables.size()
                        + ") does not " + "match expected number of elements (schema: " + inputSchema.numColumns()
                        + "). Transform = " + toString());
    }
    int idx = getColumnIdx();

    int n = stateNames.size();
    List<Writable> out = new ArrayList<>(writables.size() + n);

    int i = 0;
    for (Writable w : writables) {

        if (i++ == idx) {
            //Do conversion
            String str = w.toString();
            Integer classIdx = statesMap.get(str);
            if (classIdx == null) {
                throw new IllegalStateException("Cannot convert categorical value to one-hot: input value (\"" + str
                        + "\") is not in the list of known categories (state names/categories: " + stateNames + ")");
            }
            for (int j = 0; j < n; j++) {
                if (j == classIdx)
                    out.add(new IntWritable(1));
                else
                    out.add(new IntWritable(0));
            }
        } else {
            //No change to this column
            out.add(w);
        }
    }
    return out;
}

Example 15

Source File: VasttextTextVectorizer.java From scava with Eclipse Public License 2.0

5 votes

protected List<String> tokensFromRecord(Writable writable)
{

	String text = writable.toString();
	Tokenizer tokenizer = tokenizerFactory.create(text);
	List<String> tokens = new ArrayList<String>();
	while (tokenizer.hasMoreTokens())
        tokens.add(tokenizer.nextToken());
	return tokens;
}

Example 16

Source File: LabelWriterConverter.java From deeplearning4j with Apache License 2.0

4 votes

@Override
public Writable convert(Writable writable) throws WritableConverterException {
    String label = writable.toString();
    return new IntWritable(labels.indexOf(label));
}

Example 17

Source File: WritableToStringFunction.java From DataVec with Apache License 2.0

4 votes

@Override
public String call(Writable writable) throws Exception {
    return writable.toString();
}

Example 18

Source File: ReplaceStringTransform.java From deeplearning4j with Apache License 2.0

4 votes

@Override
public Text map(final Writable writable) {
    String value = writable.toString();
    value = replaceAll(value);
    return new Text(value);
}

Example 19

Source File: TypeConversion.java From DataVec with Apache License 2.0

4 votes

public String convertString(Writable writable) {
    return writable.toString();
}

Example 20

Source File: ConvertToString.java From deeplearning4j with Apache License 2.0

2 votes

/**
 * Transform the writable in to a
 * string
 *
 * @param writable the writable to transform
 * @return the string form of this writable
 */
@Override
public Text map(Writable writable) {
    return new Text(writable.toString());
}