com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage Java Examples

/**
 * Migrates data into the relevant {@code component} tables.
 */
public void extractComponents() throws SQLException, IOException { // NOSONAR

  addRidColumns("component");

  Map<String, PreparedStatement> componentStatements = prepareBatch(
      "component", "repository_id", "namespace", "name", "version", "attributes", "rid");

  try {
    int componentCount = 0;
    long expectedComponentCount = componentDb.countClass("component");
    for (ODocument component : componentDb.browseClass("component")) {

      OIdentifiable bucket = component.field("bucket", OType.LINK);
      String format = normalizeFormat(component.field("format", OType.STRING));
      Map<?, ?> attributes = component.field("attributes", OType.EMBEDDEDMAP);
      String group = component.field("group", OType.STRING);
      String name = component.field("name", OType.STRING);
      String version = component.field("version", OType.STRING);

      insert(componentStatements.get(format), bucketRepositoryIds.get(bucket.getIdentity()),
          nullToEmpty(group), name, nullToEmpty(version), json(attributes), component);

      if (++componentCount % BATCH_SIZE == 0) {
        executeBatch(componentStatements);

        log.info("...migrating {}/{} components", componentCount, expectedComponentCount);

        // force clear as we won't revisit these records and it keeps memory use down
        ((OAbstractPaginatedStorage) componentDb.getStorage()).getReadCache().clear();
      }
    }

    executeBatch(componentStatements);

    log.info("Migrated {} components", componentCount);
  }
  finally {
    closeBatch(componentStatements);
  }
}
 

/**
 * Migrates data into the relevant {@code asset} tables.
 */
public void extractAssets() throws SQLException, IOException { // NOSONAR

  addRidColumns("asset");
  addColumn("asset", "component_rid");

  Map<String, PreparedStatement> assetStatements = prepareBatch(
      "asset", "repository_id", "component_id", "asset_blob_id", "path", "last_downloaded", "attributes", "rid", "component_rid");

  Map<String, PreparedStatement> assetBlobStatements = prepareBatch(
      "asset_blob", "blob_ref", "blob_size", "content_type", "blob_created", "created_by", "created_by_ip");

  try {
    int assetCount = 0;
    Map<String, Integer> blobCountsPerFormat = new HashMap<>();
    long expectedAssetCount = componentDb.countClass("asset");
    for (ODocument asset : componentDb.browseClass("asset")) {

      OIdentifiable bucket = asset.field("bucket", OType.LINK);
      String format = normalizeFormat(asset.field("format", OType.STRING));
      Map<?, ?> attributes = asset.field("attributes", OType.EMBEDDEDMAP);
      OIdentifiable component = asset.field("component", OType.LINK);
      String name = asset.field("name", OType.STRING);
      long size = asset.field("size", OType.LONG);
      String contentType = asset.field("content_type", OType.STRING);
      String blobRef = asset.field("blob_ref", OType.STRING);
      Date lastDownloaded = asset.field("last_downloaded", OType.DATETIME);
      Date blobCreated = asset.field("blob_created", OType.DATETIME);
      Date blobUpdated = asset.field("blob_updated", OType.DATETIME);
      String createdBy =  asset.field("created_by", OType.STRING);
      String createdByIp = asset.field("created_by_ip", OType.STRING);

      Integer assetBlobId = null;
      if (blobRef != null) {
        insert(assetBlobStatements.get(format), blobRef, size, contentType,
            blobUpdated != null ? blobUpdated : blobCreated, createdBy, createdByIp);

        assetBlobId = blobCountsPerFormat.merge(format, 1, Integer::sum);
      }

      insert(assetStatements.get(format), bucketRepositoryIds.get(bucket.getIdentity()),
          null, assetBlobId, normalizePath(name), lastDownloaded, json(attributes), asset, component);

      if (++assetCount % BATCH_SIZE == 0) {
        executeBatch(assetBlobStatements);
        executeBatch(assetStatements);

        log.info("...migrating {}/{} assets", assetCount, expectedAssetCount);

        // force clear as we won't revisit these records and it keeps memory use down
        ((OAbstractPaginatedStorage) componentDb.getStorage()).getReadCache().clear();
      }
    }

    executeBatch(assetBlobStatements);
    executeBatch(assetStatements);

    log.info("Migrated {} assets", assetCount);
  }
  finally {
    closeBatch(assetBlobStatements);
    closeBatch(assetStatements);
  }

  log.info("Linking assets to components (this may take a while)");

  linkAssetsToComponents();
}
 

public void extractBrowseNodes() throws SQLException {
  addColumn("browse_node", "asset_rid", true);
  addColumn("browse_node", "component_rid", true);
  // for matching parent_id
  addColumn("browse_node", "parent_path", false);
  addColumn("browse_node", "parent_parent_path", false);
  addColumn("browse_node", "parent_name", false);

  createBrowseNodeParentIndex();

  Map<String, PreparedStatement> browseNodeStatements = prepareBatch(
      "browse_node", "repository_id", "format", "path", "name", "asset_rid", "component_rid", "parent_path", "parent_parent_path", "parent_name");

  try {
    int browseNodeCount = 0;

    long expectedBrowseNodeCount = componentDb.countClass("browse_node");
    for (ODocument browseNode : componentDb.browseClass("browse_node")) {
      String format = normalizeFormat(browseNode.field("format", OType.STRING));
      String repositoryName = browseNode.field("repository_name", OType.STRING);

      if (repositoryName != null) {
        ORID rid = repositoryNameToId.get(repositoryName);
        int repositoryId = bucketRepositoryIds.get(rid);

        String path = browseNode.field("path", OType.STRING);
        String name = browseNode.field("name", OType.STRING);

        String parentPath = browseNode.field("parent_path", OType.STRING);

        String parentParentPath = null;
        String parentName = null;

        if (!parentPath.equals("/")) {
          // remove trailing slash
          String parentPathMatch = parentPath.substring(0, parentPath.length() - 1);

          int lastSlash = parentPathMatch.lastIndexOf('/');
          parentParentPath = parentPathMatch.substring(0, lastSlash + 1);
          parentName = parentPathMatch.substring(lastSlash + 1);
        }

        OIdentifiable asset = browseNode.field("asset_id", OType.LINK);
        OIdentifiable component = browseNode.field("component_id", OType.LINK);

        insert(browseNodeStatements.get(format), repositoryId, format, path, name, asset, component,
            parentPath, parentParentPath, parentName);

        if (++browseNodeCount % BATCH_SIZE == 0) {
          executeBatch(browseNodeStatements);

          log.info("...migrating {}/{} browse nodes", browseNodeCount, expectedBrowseNodeCount);

          // force clear as we won't revisit these records and it keeps memory use down
          ((OAbstractPaginatedStorage) componentDb.getStorage()).getReadCache().clear();
        }
      }
    }

    executeBatch(browseNodeStatements);

    log.info("Migrated {} browse nodes", browseNodeCount);
  }
  finally {
    closeBatch(browseNodeStatements);
  }

  log.info("Linking browse nodes to assets (this may take a while)");

  linkBrowseNodesToAssets();

  log.info("Linking browse nodes to components (this may take a while)");

  linkBrowseNodesToComponents();

  log.info("Linking browse nodes to their parents (this may take a while)");

  linkBrowseNodesToParent();

  dropBrowseNodeParentIndex();
}