us.codecraft.webmagic.model.annotation.ExtractBy Java Examples

The following examples show how to use us.codecraft.webmagic.model.annotation.ExtractBy. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: ExtractorUtils.java From webmagic with Apache License 2.0

6 votes

public static Selector getSelector(ExtractBy extractBy) {
    String value = extractBy.value();
    Selector selector;
    switch (extractBy.type()) {
        case Css:
            selector = new CssSelector(value);
            break;
        case Regex:
            selector = new RegexSelector(value);
            break;
        case XPath:
            selector = new XpathSelector(value);
            break;
        case JsonPath:
            selector = new JsonPathSelector(value);
            break;
        default:
            selector = new XpathSelector(value);
    }
    return selector;
}

Example #2

Source File: ModelPipeline.java From webmagic with Apache License 2.0

6 votes

@Override
public void process(ResultItems resultItems, Task task) {
    for (Map.Entry<Class, PageModelPipeline> classPageModelPipelineEntry : pageModelPipelines.entrySet()) {
        Object o = resultItems.get(classPageModelPipelineEntry.getKey().getCanonicalName());
        if (o != null) {
            Annotation annotation = classPageModelPipelineEntry.getKey().getAnnotation(ExtractBy.class);
            if (annotation == null || !((ExtractBy) annotation).multi()) {
                classPageModelPipelineEntry.getValue().process(o, task);
            } else {
                List<Object> list = (List<Object>) o;
                for (Object o1 : list) {
                    classPageModelPipelineEntry.getValue().process(o1, task);
                }
            }
        }
    }
}

Example #3

Source File: ExtractorUtils.java From webmagic with Apache License 2.0

5 votes

public static List<Selector> getSelectors(ExtractBy[] extractBies) {
    List<Selector> selectors = new ArrayList<Selector>();
    if (extractBies == null) {
        return selectors;
    }
    for (ExtractBy extractBy : extractBies) {
        selectors.add(getSelector(extractBy));
    }
    return selectors;
}

Example #4

Source File: PageModelCollectorPipeline.java From webmagic with Apache License 2.0

5 votes

@Override
public synchronized void process(ResultItems resultItems, Task task) {
    Object o = resultItems.get(clazz.getCanonicalName());
    if (o != null) {
        Annotation annotation = clazz.getAnnotation(ExtractBy.class);
        if (annotation == null || !((ExtractBy) annotation).multi()) {
            classPipeline.process((T) o, task);
        } else {
            List<Object> list = (List<Object>) o;
            for (Object o1 : list) {
               classPipeline.process((T) o1, task);
            }
        }
    }
}