org.apache.tika.mime.MimeTypes Java Examples

/**
 * Returns the top tika mime-type definition for the media type.
 * WARN: this only returns explicit defined mime-types (canonical), NOT aliases.
 * if exact true, will double-check that parameters match as well (not guaranteed by MimeTypes.getRegisteredMimeType).
 * FIXME: exact doesn't handle parameter order.
 */
public static MimeType getMimeTypeForMediaTypeSafe(String mediaType, MimeTypes mimeTypes, boolean exact) {
    try {
        MimeType mimeType = mimeTypes.getRegisteredMimeType(mediaType);
        if (mimeType != null && exact) {
            // NOTE: because the way getRegisteredMimeType works, it may return non-null
            // even if not exact name match, due to parameters.
            // FIXME: this check won't handle parameter order difference
            // also check if another normalize call would be more appropriate...
            if (!getMimeTypeId(mediaType).equals(getMimeTypeId(mimeType.getName()))) {
                return null;
            }
        }
        return mimeType;
    } catch (MimeTypeException e) {
        return null;
    }
}
 

public static String extensionForMIME(String mimeType) {
    if (mimeType.isEmpty())
        return DEFAULT_EXT;

    MimeType mime = null;
    try {
        mime = MimeTypes.getDefaultMimeTypes().forName(mimeType);
    } catch (MimeTypeException ex) {
        LOGGER.log(Level.WARNING, "can't find mimetype", ex);
    }

    String m = mime != null ? mime.getExtension() : "";
    // remove dot
    if (!m.isEmpty())
        m = m.substring(1);
    return StringUtils.defaultIfEmpty(m, DEFAULT_EXT);
}
 

public static String getMediaTypeDescriptionOrNullSafe(MediaType mediaType, MimeTypes mimeTypes) {
    MimeType mimeType = getMimeTypeForMediaTypeSafe(mediaType, mimeTypes, true);
    String description = null;

    if (mimeType != null) {
        description = mimeType.getDescription();
    } else {
        // when this prints, it's because of imperfections in tika-mimetypes.xml...
        Debug.logWarning("No Tika mime-type for MediaType: " + mediaType.toString(), module);
    }
    if (UtilValidate.isNotEmpty(description)) {
        return description;
    } else {
        MediaTypeRegistry registry = mimeTypes.getMediaTypeRegistry();

        // check if can find one in supertype
        MediaType superType = registry.getSupertype(mediaType);
        if (superType != null) {
            description = getMediaTypeDescriptionOrNullSafe(superType, mimeTypes);
            if (UtilValidate.isNotEmpty(description)) {
                return description + " (sub-type)";
            }
        }
    }
    return null;
}
 

/**
 * Uses extension to mime types cache backed by {@link NexusMimeTypes} and Tika registry.
 */
@Nonnull
private List<String> guessMimeTypesListFromPath(final String path) {
  checkNotNull(path);
  final String pathExtension = Files.getFileExtension(path);
  try {
    final List<String> mimeTypes = new ArrayList<>();
    List<String> extBasedTypes = extensionToMimeTypeCache.get(pathExtension);
    if (extBasedTypes != null && !extBasedTypes.isEmpty()) {
      mimeTypes.addAll(extBasedTypes);
    }
    if (mimeTypes.isEmpty()) {
      mimeTypes.add(MimeTypes.OCTET_STREAM);
    }
    return mimeTypes;
  }
  catch (ExecutionException e) {
    throw new RuntimeException(e);
  }
}
 

@Nonnull
@Override
public List<String> detectMimeTypes(final InputStream input, @Nullable final String fileName) throws IOException {
  checkNotNull(input);

  List<String> detected = Lists.newArrayList();
  Metadata metadata = new Metadata();
  if (fileName != null) {
    metadata.set(Metadata.RESOURCE_NAME_KEY, fileName);
  }

  MediaType mediaType;
  try (final TikaInputStream tis = TikaInputStream.get(input)) {
    mediaType = detector.detect(tis, metadata);
  }

  // unravel to least specific
  unravel(detected, mediaType);

  if (detected.isEmpty()) {
    detected.add(MimeTypes.OCTET_STREAM);
  }

  return detected;
}
 

/**
 * Control whether the uploaded files are of HTML type and whether their
 * size is under the maxFileSize limit.
 *
 * @param uploadAuditSetUpCommand
 * @param errors
 */
private void validateFiles(AuditSetUpCommand uploadAuditSetUpCommand, Errors errors) {
    boolean emptyFile = true;
    Metadata metadata = new Metadata();
    MimeTypes mimeTypes = TikaConfig.getDefaultConfig().getMimeRepository();
    String mime;

    for (int i=0;i<uploadAuditSetUpCommand.getFileInputList().length;i++ ) {
        try {
            CommonsMultipartFile cmf = uploadAuditSetUpCommand.getFileInputList()[i];
            if (cmf.getSize() > maxFileSize) {
                Long maxFileSizeInMega = maxFileSize / 1000000;
                String[] arg = {maxFileSizeInMega.toString()};
                errors.rejectValue(ID_INPUT_FILE_PREFIX + "[" + i + "]", FILE_SIZE_EXCEEDED_MSG_BUNDLE_KEY, arg, "{0}");
            }
            if (cmf.getSize() > 0) {
                emptyFile = false;
                mime = mimeTypes.detect(new BufferedInputStream(cmf.getInputStream()), metadata).toString();
                LOGGER.debug("mime  " + mime + "  " +cmf.getOriginalFilename());
                if (!authorizedMimeType.contains(mime)) {
                    errors.rejectValue(ID_INPUT_FILE_PREFIX + "[" + i + "]", NOT_HTML_MSG_BUNDLE_KEY);
                }
            }
        } catch (IOException ex) {
            LOGGER.warn(ex);
            errors.rejectValue(ID_INPUT_FILE_PREFIX + "[" + i + "]", NOT_HTML_MSG_BUNDLE_KEY);
        }
    }
    if(emptyFile) { // if no file is uploaded
        LOGGER.debug("emptyFiles");
        errors.rejectValue(GENERAL_ERROR_MSG_KEY,
                NO_FILE_UPLOADED_MSG_BUNDLE_KEY);
    }
}
 

/**
 * Generate attachment extension from mime type
 *
 * @param type valid mime-type
 * @return extension if it's known for specified mime-type, or empty string
 * otherwise
 */
public static String getExtensionByMimeType(String type) {
    MimeTypes types = getDefaultMimeTypes();
    try {
        return types.forName(type).getExtension();
    } catch (Exception e) {
        LOGGER.warn("Can't detect extension for MIME-type " + type, e);
        return "";
    }
}
 

/**
 * WARN: very slow, not for heavy use.
 */
public static List<GenericValue> makeEntityMimeTypes(Delegator delegator, Collection<MediaType> mediaTypes, MimeTypes mimeTypes, boolean aliases, boolean missingOnly) throws GenericEntityException {
    List<GenericValue> mimeTypeValues = new ArrayList<>(mediaTypes.size());
    for(MediaType mediaType : mediaTypes) {
        GenericValue mainValue = makeEntityMimeType(delegator, mediaType, mimeTypes, "", getMimeTypeId(mediaType));
        if (!missingOnly || UtilValidate.isEmpty(delegator.findOne("MimeType", true, UtilMisc.toMap("mimeTypeId", getMimeTypeId(mediaType))))) {
            mimeTypeValues.add(mainValue);
        }
        if (aliases) {
            Set<MediaType> aliasSet = getMediaTypeAliases(mediaType, mimeTypes.getMediaTypeRegistry());
            for(MediaType alias : aliasSet) {
                GenericValue aliasValue = makeEntityMimeType(delegator, alias, mimeTypes,
                        mainValue.getString("description") + " (alias)",
                        getMimeTypeId(alias) + " (alias for " + getMimeTypeId(mediaType) + ")");
                if (!missingOnly || UtilValidate.isEmpty(delegator.findOne("MimeType", true, UtilMisc.toMap("mimeTypeId", getMimeTypeId(alias))))) {
                    mimeTypeValues.add(aliasValue);
                }

                // SANITY CHECK
                Set<MediaType> aliasAliasSet = getMediaTypeAliases(alias, mimeTypes.getMediaTypeRegistry());
                if (aliasAliasSet.isEmpty()) {
                    Debug.logWarning("Tika: Sanity check failed: " + alias.toString() + " has no aliases (unlike its canonical form)", module);
                } else if (aliasSet.size() != aliasAliasSet.size()) {
                    Debug.logWarning("Tika: Sanity check failed: " + alias.toString() + " has different number of aliases than its canonical form", module);
                }
            }
        }
    }
    return mimeTypeValues;
}
 

public static String getMediaTypeDescriptionAlwaysSafe(MediaType mediaType, MimeTypes mimeTypes, String overrideDesc, String defaultDesc) {
    if (UtilValidate.isNotEmpty(overrideDesc)) {
        return overrideDesc;
    }
    String description = getMediaTypeDescriptionOrNullSafe(mediaType, mimeTypes);
    if (UtilValidate.isNotEmpty(description)) {
        return description;
    } else {
        return defaultDesc;
    }
}
 

public static GenericValue makeEntityMimeType(Delegator delegator, MediaType mediaType, MimeTypes mimeTypes, String overrideDesc, String defaultDesc) throws GenericEntityException {
    return delegator.makeValue("MimeType",
            "mimeTypeId", getMimeTypeId(mediaType),
            "description", getMediaTypeDescriptionAlwaysSafe(mediaType, mimeTypes, overrideDesc, defaultDesc));
}
 

public static MimeTypes getMimeTypeRegistry() {
    return MimeTypes.getDefaultMimeTypes(); // TODO?: unhardcode registry selection;
}
 

/**
 * 
 * @param addScenarioCommand
 * @param errors 
 * @return  whether the scenario handled by the current AddScenarioCommand
 * has a correct type and size
 */
public boolean checkScenarioFileTypeAndSize(
        AddScenarioCommand addScenarioCommand, 
        Errors errors) {
    if (addScenarioCommand.getScenarioFile() == null) { // if no file uploaded
        LOGGER.debug("empty Scenario File");
        errors.rejectValue(GENERAL_ERROR_MSG_KEY,
                MANDATORY_FIELD_MSG_BUNDLE_KEY);
        errors.rejectValue(SCENARIO_FILE_KEY,
                NO_SCENARIO_UPLOADED_MSG_BUNDLE_KEY);
        return false;
    }
    Metadata metadata = new Metadata();
    MimeTypes mimeTypes = TikaConfig.getDefaultConfig().getMimeRepository();
    String mime;
    try {
        CommonsMultipartFile cmf = addScenarioCommand.getScenarioFile();
        if (cmf.getSize() > maxFileSize) {
            Long maxFileSizeInMega = maxFileSize / 1000000;
            String[] arg = {maxFileSizeInMega.toString()};
            errors.rejectValue(GENERAL_ERROR_MSG_KEY,
                    MANDATORY_FIELD_MSG_BUNDLE_KEY);
            errors.rejectValue(SCENARIO_FILE_KEY, FILE_SIZE_EXCEEDED_MSG_BUNDLE_KEY, arg, "{0}");
            return false;
        } else if (cmf.getSize() > 0) {
            mime = mimeTypes.detect(new BufferedInputStream(cmf.getInputStream()), metadata).toString();
            LOGGER.debug("mime  " + mime + "  " + cmf.getOriginalFilename());
            if (!authorizedMimeType.contains(mime)) {
                errors.rejectValue(GENERAL_ERROR_MSG_KEY,
                    MANDATORY_FIELD_MSG_BUNDLE_KEY);
                errors.rejectValue(SCENARIO_FILE_KEY, NOT_SCENARIO_MSG_BUNDLE_KEY);
                return false;
            }
        } else {
            LOGGER.debug("File with size null");
            errors.rejectValue(GENERAL_ERROR_MSG_KEY,
                MANDATORY_FIELD_MSG_BUNDLE_KEY);
            errors.rejectValue(SCENARIO_FILE_KEY,
                NO_SCENARIO_UPLOADED_MSG_BUNDLE_KEY);
            return false;
        }
    } catch (IOException ex) {
        LOGGER.warn(ex);
        errors.rejectValue(SCENARIO_FILE_KEY, NOT_SCENARIO_MSG_BUNDLE_KEY);
        errors.rejectValue(GENERAL_ERROR_MSG_KEY,
                MANDATORY_FIELD_MSG_BUNDLE_KEY);
        return false;
    }
    return true;
}
 

/**
 * A facade interface to trying all the possible mime type resolution
 * strategies available within Tika. First, the mime type provided in
 * <code>typeName</code> is cleaned, with {@link #cleanMimeType(String)}.
 * Then the cleaned mime type is looked up in the underlying Tika
 * {@link MimeTypes} registry, by its cleaned name. If the {@link MimeType} is
 * found, then that mime type is used, otherwise URL resolution is
 * used to try and determine the mime type. If that means is unsuccessful, and
 * if <code>mime.type.magic</code> is enabled in {@link NutchConfiguration},
 * then mime type magic resolution is used to try and obtain a
 * better-than-the-default approximation of the {@link MimeType}.
 * 
 * @param typeName
 *          The original mime type, returned from a {@link ProtocolOutput}.
 * @param url
 *          The given @see url, that Nutch was trying to crawl.
 * @param data
 *          The byte data, returned from the crawl, if any.
 * @return The correctly, automatically guessed {@link MimeType} name.
 */
public String autoResolveContentType(String typeName, String url, byte[] data) {
  String retType = null;
  String magicType = null;
  MimeType type = null;
  String cleanedMimeType = null;

  try {
    cleanedMimeType = MimeUtil.cleanMimeType(typeName) != null ? this.mimeTypes
        .forName(MimeUtil.cleanMimeType(typeName)).getName()
        : null;
  } catch (MimeTypeException mte) {
    // Seems to be a malformed mime type name...
  }

  // first try to get the type from the cleaned type name
  try {
    type = cleanedMimeType != null ? this.mimeTypes.forName(cleanedMimeType)
        : null;
  } catch (MimeTypeException e) {
    type = null;
  }

  // if returned null, or if it's the default type then try url resolution
  if (type == null
      || (type != null && type.getName().equals(MimeTypes.OCTET_STREAM))) {
    // If no mime-type header, or cannot find a corresponding registered
    // mime-type, then guess a mime-type from the url pattern
    type = this.mimeTypes.getMimeType(url) != null ? this.mimeTypes
        .getMimeType(url) : type;
  }

  retType= type.getName();

  // if magic is enabled use mime magic to guess if the mime type returned
  // from the magic guess is different than the one that's already set so far
  // if it is, and it's not the default mime type, then go with the mime type
  // returned by the magic
  if (this.mimeMagic) {
    magicType = tika.detect(data);

    // Deprecated in Tika 1.0 See https://issues.apache.org/jira/browse/NUTCH-1230
    //MimeType magicType = this.mimeTypes.getMimeType(data);
    if (magicType != null && !magicType.equals(MimeTypes.OCTET_STREAM)
        && !magicType.equals(MimeTypes.PLAIN_TEXT)
        && retType != null && !retType.equals(magicType)) {

      // If magic enabled and the current mime type differs from that of the
      // one returned from the magic, take the magic mimeType
      retType = magicType;
    }

    // if type is STILL null after all the resolution strategies, go for the
    // default type
    if (retType == null) {
      try {
        retType = MimeTypes.OCTET_STREAM;
      } catch (Exception ignore) {
      }
    }
  }

  return retType;
}
 

/** Unit tests for getContentType(String, String, byte[]) method. */
public void testGetContentType() throws Exception {
  Content c = null;
  Metadata p = new Metadata();

  c = new Content("http://www.foo.com/",
                  "http://www.foo.com/",
                  "".getBytes("UTF8"),
                  "text/html; charset=UTF-8", p, conf);
  assertEquals("text/html", c.getContentType());

  c = new Content("http://www.foo.com/foo.html",
                  "http://www.foo.com/",
                  "".getBytes("UTF8"),
                  "", p, conf);
  assertEquals("text/html", c.getContentType());

  c = new Content("http://www.foo.com/foo.html",
                  "http://www.foo.com/",
                  "".getBytes("UTF8"),
                  null, p, conf);
  assertEquals("text/html", c.getContentType());

  c = new Content("http://www.foo.com/",
                  "http://www.foo.com/",
                  "<html></html>".getBytes("UTF8"),
                  "", p, conf);
  assertEquals("text/html", c.getContentType());

  c = new Content("http://www.foo.com/foo.html",
                  "http://www.foo.com/",
                  "<html></html>".getBytes("UTF8"),
                  "text/plain", p, conf);
  assertEquals("text/html", c.getContentType());

  c = new Content("http://www.foo.com/foo.png",
                  "http://www.foo.com/",
                  "<html></html>".getBytes("UTF8"),
                  "text/plain", p, conf);
  assertEquals("text/html", c.getContentType());

  c = new Content("http://www.foo.com/",
                  "http://www.foo.com/",
                  "".getBytes("UTF8"),
                  "", p, conf);
  assertEquals(MimeTypes.OCTET_STREAM, c.getContentType());

  c = new Content("http://www.foo.com/",
                  "http://www.foo.com/",
                  "".getBytes("UTF8"),
                  null, p, conf);
  assertNotNull(c.getContentType());
}
 

/** Unit tests for getContentType(String, String, byte[]) method. */
public void testGetContentType() throws Exception {
  Content c = null;
  Metadata p = new Metadata();

  c = new Content("http://www.foo.com/",
                  "http://www.foo.com/",
                  "".getBytes("UTF8"),
                  "text/html; charset=UTF-8", p, conf);
  assertEquals("text/html", c.getContentType());

  c = new Content("http://www.foo.com/foo.html",
                  "http://www.foo.com/",
                  "".getBytes("UTF8"),
                  "", p, conf);
  assertEquals("text/html", c.getContentType());

  c = new Content("http://www.foo.com/foo.html",
                  "http://www.foo.com/",
                  "".getBytes("UTF8"),
                  null, p, conf);
  assertEquals("text/html", c.getContentType());

  c = new Content("http://www.foo.com/",
                  "http://www.foo.com/",
                  "<html></html>".getBytes("UTF8"),
                  "", p, conf);
  assertEquals("text/html", c.getContentType());

  c = new Content("http://www.foo.com/foo.html",
                  "http://www.foo.com/",
                  "<html></html>".getBytes("UTF8"),
                  "text/plain", p, conf);
  assertEquals("text/html", c.getContentType());

  c = new Content("http://www.foo.com/foo.png",
                  "http://www.foo.com/",
                  "<html></html>".getBytes("UTF8"),
                  "text/plain", p, conf);
  assertEquals("text/html", c.getContentType());

  c = new Content("http://www.foo.com/",
                  "http://www.foo.com/",
                  "".getBytes("UTF8"),
                  "", p, conf);
  assertEquals(MimeTypes.OCTET_STREAM, c.getContentType());

  c = new Content("http://www.foo.com/",
                  "http://www.foo.com/",
                  "".getBytes("UTF8"),
                  null, p, conf);
  assertNotNull(c.getContentType());
}
 

/**
 * Tests that {@link ReferenceResource reference resources} are marshalled to
 * the expected XML format.
 * @throws IOException if the {@link Diff} constructor fails
 * @throws JAXBException if the {@link JAXBContext} or {@link Marshaller} fail
 * @throws MimeTypeException if {@link MimeTypes#forName(String)} fails
 * @throws SAXException if the {@link Diff} constructor fails
 */
@Test
public void testXmlMarshalling() throws IOException, JAXBException,
  MimeTypeException, SAXException
{
  String productId = "123";
  int refIndex = 0;

  // Create a new ReferenceResource using a Reference instance.
  Reference reference = new Reference("original", "dataStore", 1000,
    new MimeTypes().forName("text/plain"));

  ReferenceResource resource = new ReferenceResource(productId, refIndex,
    reference, new File("/tmp"));


  // Generate the expected output.
  String expectedXml =
      "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>"
    + "<reference>"
    + "<productId>" + productId + "</productId>"
    + "<refIndex>" + refIndex + "</refIndex>"
    + "<dataStoreReference>"
    +    reference.getDataStoreReference()
    + "</dataStoreReference>"
    + "<originalReference>"
    +    reference.getOrigReference()
    + "</originalReference>"
    + "<mimeType>" + reference.getMimeType().getName() + "</mimeType>"
    + "<fileSize>" + reference.getFileSize() +  "</fileSize>"
    + "</reference>";


  // Set up a JAXB context and marshall the ReferenceResource to XML.
  JAXBContext context = JAXBContext.newInstance(resource.getClass());
  Marshaller marshaller = context.createMarshaller();
  StringWriter writer = new StringWriter();
  marshaller.marshal(resource, writer);

  // Compare the expected and actual outputs.
  XMLUnit.setIgnoreWhitespace(true);
  XMLUnit.setIgnoreComments(true);
  XMLUnit.setIgnoreAttributeOrder(true);
  Diff diff = new Diff(expectedXml, writer.toString());
  assertTrue("The output XML was different to the expected XML: "
    + diff.toString(), diff.identical());
}
 

/**
 * Tests that {@link ProductResource product resources} are marshalled to
 * the expected XML format.
 * @throws IOException if the {@link Diff} constructor fails
 * @throws JAXBException if the {@link JAXBContext} or {@link Marshaller} fail
 * @throws MimeTypeException if {@link MimeTypes#forName(String)} fails
 * @throws SAXException if the {@link Diff} constructor fails
 */
@Test
public void testXmlMarshalling() throws IOException, JAXBException,
  MimeTypeException, SAXException
{
  // Create a ProductResource using ProductType, Reference, Metadata and
  // Product instances.
  Hashtable metadataEntries = new Hashtable<String, Object>();
  metadataEntries.put("CAS.Test", "test value");
  Metadata metadata = new Metadata();
  metadata.addMetadata(metadataEntries);

  Reference reference = new Reference("original", "dataStore", 1000,
    new MimeTypes().forName("text/plain"));
  List<Reference> references = new ArrayList<Reference>();
  references.add(reference);

  ProductType productType = new ProductType("1", "GenericFile", "test type",
    "repository", "versioner");

  Product product = new Product();
  product.setProductId("123");
  product.setProductName("test.txt");
  product.setProductStructure(Product.STRUCTURE_FLAT);
  product.setProductType(productType);

  ProductResource resource = new ProductResource(product, metadata,
    references, new File("/tmp"));


  // Generate the expected output.
  String expectedXml =
      "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>"
      + "<product>"
      + "<id>" + product.getProductId() + "</id>"
      + "<name>" + product.getProductName() + "</name>"
      + "<structure>" + product.getProductStructure() + "</structure>"
      + "<type>" + productType.getName() + "</type>"
      + "<metadata>"
      + "<keyval>"
      + "<key>" + metadata.getAllKeys().get(0) + "</key>"
      + "<val>" + metadata.getAllValues().get(0) + "</val>"
      + "</keyval>"
      + "</metadata>"
      + "<references>"
      + "<reference>"
      + "<productId>" + product.getProductId() + "</productId>"
      + "<refIndex>0</refIndex>"
      + "<dataStoreReference>"
      +    reference.getDataStoreReference()
      + "</dataStoreReference>"
      + "<originalReference>"
      +    reference.getOrigReference()
      + "</originalReference>"
      + "<mimeType>" + reference.getMimeType().getName() + "</mimeType>"
      + "<fileSize>" + reference.getFileSize() + "</fileSize>"
      + "</reference>"
      + "</references>"
      + "</product>";


  // Set up a JAXB context and marshall the ProductResource to XML.
  JAXBContext context = JAXBContext.newInstance(resource.getClass());
  Marshaller marshaller = context.createMarshaller();
  StringWriter writer = new StringWriter();
  marshaller.marshal(resource, writer);

  // Compare the expected and actual outputs.
  XMLUnit.setIgnoreWhitespace(true);
  XMLUnit.setIgnoreComments(true);
  XMLUnit.setIgnoreAttributeOrder(true);
  Diff diff = new Diff(expectedXml, writer.toString());
  assertTrue("The output XML was different to the expected XML: "
    + diff.toString(), diff.identical());
}
 

/**
 * Tests that {@link TransferResource transfer resources} are marshalled to
 * the expected XML format.
 * @throws IOException if the {@link Diff} constructor fails
 * @throws JAXBException if the {@link JAXBContext} or {@link Marshaller} fail
 * @throws MimeTypeException if {@link MimeTypes#forName(String)} fails
 * @throws SAXException if the {@link Diff} constructor fails
 */
@Test
public void testXmlMarshalling() throws IOException, JAXBException,
  MimeTypeException, SAXException
{
  // Create a TransferResource using ProductType, Product, Metadata, Reference
  // and FileTransferStatus instances.
  Hashtable metadataEntries = new Hashtable<String, Object>();
  metadataEntries.put("CAS.ProductReceivedTime", "2013-09-12T16:25:50.662Z");
  Metadata metadata = new Metadata();
  metadata.addMetadata(metadataEntries);

  Reference reference = new Reference("original", "dataStore", 1000,
    new MimeTypes().forName("text/plain"));

  ProductType productType = new ProductType("1", "GenericFile", "test type",
    "repository", "versioner");

  Product product = new Product();
  product.setProductId("123");
  product.setProductName("test product");
  product.setProductStructure(Product.STRUCTURE_FLAT);
  product.setProductType(productType);

  FileTransferStatus status = new FileTransferStatus(reference, 1000, 100,
    product);

  TransferResource resource = new TransferResource(product, metadata, status);


  // Generate the expected output.
  String expectedXml =
    "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>"
    + "<transfer>"
    + "<productName>" + product.getProductName() + "</productName>"
    + "<productId>" + product.getProductId() + "</productId>"
    + "<productTypeName>" + productType.getName() + "</productTypeName>"
    + "<dataStoreReference>"
    +   reference.getDataStoreReference()
    + "</dataStoreReference>"
    + "<origReference>"
    +   reference.getOrigReference()
    + "</origReference>"
    + "<mimeType>" + reference.getMimeType().getName() + "</mimeType>"
    + "<fileSize>" + reference.getFileSize() + "</fileSize>"
    + "<totalBytes>" + reference.getFileSize() + "</totalBytes>"
    + "<bytesTransferred>"
    +    status.getBytesTransferred()
    + "</bytesTransferred>"
    + "<percentComplete>"
    +    status.computePctTransferred() * 100
    + "</percentComplete>"
    + "<productReceivedTime>"
    +    metadata.getAllValues().get(0)
    + "</productReceivedTime>"
    + "</transfer>";

  // Set up a JAXB context and marshall the DatasetResource to XML.
  JAXBContext context = JAXBContext.newInstance(resource.getClass());
  Marshaller marshaller = context.createMarshaller();
  StringWriter writer = new StringWriter();
  marshaller.marshal(resource, writer);

  // Compare the expected and actual outputs.
  XMLUnit.setIgnoreWhitespace(true);
  XMLUnit.setIgnoreComments(true);
  XMLUnit.setIgnoreAttributeOrder(true);
  Diff diff = new Diff(expectedXml, writer.toString());
  assertTrue("The output XML was different to the expected XML: "
    + diff.toString(), diff.identical());
}
 

/**
 * @param args
 */
public static void main(String[] args) throws DataTransferException,
      IOException, URISyntaxException {
   String usage = "LocalFileTransfer --productName <name> --productRepo <repo> [--dir <dirRef>] [--files <origRef 1>...<origRef N>]\n";

   MimeTypes mimeTypeRepo;
   try {
      mimeTypeRepo = MimeTypesFactory
            .create(System
                  .getProperty("org.apache.oodt.cas.filemgr.mime.type.repository"));
   } catch (MimeTypeException e) {
      LOG.log(Level.SEVERE, e.getMessage());
      throw new IOException(e.getMessage());
   }

   String productName = null;
   String productRepo = null;
   String transferType = null;
   Reference dirReference = null;

   List<Reference> fileReferences = null;

   for (int i = 0; i < args.length; i++) {
      if (args[i].equals("--dir")) {
         transferType = "dir";
         dirReference = new Reference();
         dirReference.setOrigReference(new File(new URI(args[++i])).toURI()
               .toString());
         LOG.log(Level.FINER,
               "LocalFileTransfer.main: Generated orig reference: "
                     + dirReference.getOrigReference());
      } else if (args[i].equals("--files")) {
         transferType = "files";
         fileReferences = new Vector<Reference>();
         for (int j = i + 1; j < args.length; j++) {
            LOG.log(Level.FINER, "LocalFileTransfer.main: Adding file ref: "
                  + args[j]);
            fileReferences.add(new Reference(args[j], null,
                  new File(args[j]).length(), mimeTypeRepo
                        .getMimeType(args[j])));
         }
      } else if (args[i].equals("--productName")) {
         productName = args[++i];
      } else if (args[i].equals("--productRepo")) {
         productRepo = args[++i];
      }
   }

   if (transferType == null || (((transferType.equals("dir") && dirReference == null) || (
       transferType.equals("files") && fileReferences == null) || (!(
       transferType.equals("dir") || transferType
           .equals("files"))) || productName == null || productRepo == null))) {
      System.err.println(usage);
      System.exit(1);
   }

   // construct a new Product
   Product p = new Product();
   p.setProductName(productName);

   if (transferType.equals("dir")) {
      p.setProductStructure(Product.STRUCTURE_HIERARCHICAL);
      dirReference.setDataStoreReference(new File(new URI(productRepo))
            .toURI().toURL().toExternalForm()
            + URLEncoder.encode(p.getProductName(), "UTF-8") + "/");
      p.getProductReferences().add(dirReference);
      /* we'll do a simple versioning scheme ourselves: no versioning! */
      p.getProductReferences().addAll(
            VersioningUtils.getReferencesFromDir(new File(new URI(
                  dirReference.getOrigReference()))));
      VersioningUtils.createBasicDataStoreRefsHierarchical(p
            .getProductReferences());
   } else if (transferType.equals("files")) {
      p.setProductStructure("Flat");
      p.getProductReferences().addAll(fileReferences);
      VersioningUtils.createBasicDataStoreRefsFlat(productName, productRepo,
            p.getProductReferences());
   }

   DataTransfer transfer = new LocalDataTransferer();
   transfer.transferProduct(p);

}
 

private JSONObject generatePostParameters(String action, String locator, String text, long defaultWait) throws JSONException, IOException, MalformedURLException, MimeTypeException {
        JSONObject result = new JSONObject();
        String picture = "";
        String extension = "";
        /**
         * Get Picture from URL and convert to Base64
         */
        if (locator != null && !"".equals(locator)) {
            URL url = new URL(locator);
            URLConnection connection = url.openConnection();

            InputStream istream = new BufferedInputStream(connection.getInputStream());

            /**
             * Get the MimeType and the extension
             */
            String mimeType = URLConnection.guessContentTypeFromStream(istream);
            MimeTypes allTypes = MimeTypes.getDefaultMimeTypes();
            MimeType mt = allTypes.forName(mimeType);
            extension = mt.getExtension();

            /**
             * Encode in Base64
             */
            byte[] bytes = IOUtils.toByteArray(istream);
            picture = Base64.encodeBase64URLSafeString(bytes);
        }
        /**
         * Build JSONObject with parameters action : Action expected to be done
         * by Sikuli picture : Picture in Base64 format text : Text to type
         * defaultWait : Timeout for the action pictureExtension : Extension for
         * Base64 decoding
         */
        result.put("action", action);
        result.put("picture", picture);
        result.put("text", text);
        result.put("defaultWait", defaultWait);
        result.put("pictureExtension", extension);
//        result.put("minSimilarity", parameterService.getParameterStringByKey("cerberus_sikuli_minSimilarity", "", null));
        return result;
    }
 

/**
 * Makes a new GenericValue MimeType for each Tika MediaType.
 * <p>
 * Does not store.
 * WARN: UNIQUE NOT GUARANTEED
 */
public static List<GenericValue> makeAllEntityMimeTypes(Delegator delegator, boolean aliases) throws GenericEntityException {
    MimeTypes mimeTypes = getMimeTypeRegistry();
    return makeEntityMimeTypes(delegator, mimeTypes.getMediaTypeRegistry().getTypes(), mimeTypes, aliases, false);
}
 

/**
 * Makes a new GenericValue MimeType for each Tika MediaType that does not have an entity MimeType in the system.
 * <p>
 * Does not store.
 * WARN: UNIQUE NOT GUARANTEED
 */
public static List<GenericValue> makeMissingEntityMimeTypes(Delegator delegator, boolean aliases) throws GenericEntityException {
    MimeTypes mimeTypes = getMimeTypeRegistry();
    return makeEntityMimeTypes(delegator, mimeTypes.getMediaTypeRegistry().getTypes(), mimeTypes, aliases, true);
}
 

/**
 * WARN: this only returns explicit defined mime-types (canonical), NOT aliases.
 * FIXME: exact doesn't handle parameter order.
 */
public static MimeType getMimeTypeForMediaTypeSafe(MediaType mediaType, MimeTypes mimeTypes, boolean exact) {
    return getMimeTypeForMediaTypeSafe(mediaType.toString(), mimeTypes, exact);
}