Java Code Examples for org.apache.solr.common.util.Base64

The following examples show how to use org.apache.solr.common.util.Base64. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   Source File: SnapshotDistribStateManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Populate this instance from a previously generated snapshot.
 * @param snapshot previous snapshot created using this class.
 * @param config optional config to override the one from snapshot, may be null
 */
public SnapshotDistribStateManager(Map<String, Object> snapshot, AutoScalingConfig config) {
  snapshot.forEach((path, value) -> {
    @SuppressWarnings({"unchecked"})
    Map<String, Object> map = (Map<String, Object>)value;
    Number version = (Number)map.getOrDefault("version", 0);
    String owner = (String)map.get("owner");
    String modeStr = (String)map.getOrDefault("mode", CreateMode.PERSISTENT.toString());
    CreateMode mode = CreateMode.valueOf(modeStr);
    byte[] bytes = null;
    if (map.containsKey("data")) {
      bytes = Base64.base64ToByteArray((String)map.get("data"));
    }
    dataMap.put(path, new VersionedData(version.intValue(), bytes, mode, owner));
  });
  if (config != null) { // overwrite existing
    VersionedData vd = new VersionedData(config.getZkVersion(), Utils.toJSON(config), CreateMode.PERSISTENT, "0");
    dataMap.put(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, vd);
  }
  if (log.isDebugEnabled()) {
    log.debug("- loaded snapshot of {} resources", dataMap.size());
  }
}
 
Example 2
Source Project: lucene-solr   Source File: CryptoKeys.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Try with all signatures and return the name of the signature that matched
 */
public String verify(String sig, ByteBuffer data) {
  exception = null;
  for (Map.Entry<String, PublicKey> entry : keys.entrySet()) {
    boolean verified;
    try {
      verified = CryptoKeys.verify(entry.getValue(), Base64.base64ToByteArray(sig), data);
      log.debug("verified {} ", verified);
      if (verified) return entry.getKey();
    } catch (Exception e) {
      exception = e;
      log.debug("NOT verified  ");
    }

  }

  return null;
}
 
Example 3
Source Project: lucene-solr   Source File: CryptoKeys.java    License: Apache License 2.0 6 votes vote down vote up
public String verify(String sig, InputStream is) {
  exception = null;
  for (Map.Entry<String, PublicKey> entry : keys.entrySet()) {
    boolean verified;
    try {
      verified = CryptoKeys.verify(entry.getValue(), Base64.base64ToByteArray(sig), is);
      log.debug("verified {} ", verified);
      if (verified) return entry.getKey();
    } catch (Exception e) {
      exception = e;
      log.debug("NOT verified  ");
    }

  }

  return null;
}
 
Example 4
Source Project: lucene-solr   Source File: CryptoKeys.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Initialize an RSA key pair from previously saved keys. The formats listed below have been tested, other formats may
 * also be acceptable but are not guaranteed to work.
 * @param privateKeyResourceName path to private key file, encoded as a PKCS#8 in a PEM file
 * @param publicKeyResourceName path to public key file, encoded as X509 in a DER file
 * @throws IOException if an I/O error occurs reading either key file
 * @throws InvalidKeySpecException if either key file is inappropriate for an RSA key
 */
public RSAKeyPair(URL privateKeyResourceName, URL publicKeyResourceName) throws IOException, InvalidKeySpecException {
  try (InputStream inPrivate = privateKeyResourceName.openStream()) {
    String privateString = new String(inPrivate.readAllBytes(), StandardCharsets.UTF_8)
        .replaceAll("-----(BEGIN|END) PRIVATE KEY-----", "");

    PKCS8EncodedKeySpec privateSpec = new PKCS8EncodedKeySpec(java.util.Base64.getMimeDecoder().decode(privateString));
    KeyFactory rsaFactory = KeyFactory.getInstance("RSA");
    privateKey = rsaFactory.generatePrivate(privateSpec);
  } catch (NoSuchAlgorithmException e) {
    throw new AssertionError("JVM spec is required to support RSA", e);
  }

  try (InputStream inPublic = publicKeyResourceName.openStream()) {
    publicKey = getX509PublicKey(inPublic.readAllBytes());
    pubKeyStr = Base64.byteArrayToBase64(publicKey.getEncoded());
  }
}
 
Example 5
Source Project: lucene-solr   Source File: JWTAuthPluginIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void infoRequestValidateXSolrAuthHeadersBlockUnknownFalse() throws Exception {
  // Re-configure cluster with other security.json, see https://issues.apache.org/jira/browse/SOLR-14196
  shutdownCluster();
  configureCluster(NUM_SERVERS)// nodes
      .withSecurityJson(TEST_PATH().resolve("security").resolve("jwt_plugin_jwk_security_blockUnknownFalse.json"))
      .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
      .withDefaultClusterProperty("useLegacyReplicaAssignment", "false")
      .configure();
  baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString();

  Map<String, String> headers = getHeaders(baseUrl + "/admin/info/system", null);
  assertEquals("Should have received 401 code", "401", headers.get("code"));
  assertEquals("Bearer realm=\"my-solr-jwt-blockunknown-false\"", headers.get("WWW-Authenticate"));
  String authData = new String(Base64.base64ToByteArray(headers.get("X-Solr-AuthData")), UTF_8);
  assertEquals("{\n" +
      "  \"scope\":\"solr:admin\",\n" +
      "  \"redirect_uris\":[],\n" +
      "  \"authorizationEndpoint\":\"http://acmepaymentscorp/oauth/auz/authorize\",\n" +
      "  \"client_id\":\"solr-cluster\"}", authData);
}
 
Example 6
Source Project: ambari-logsearch   Source File: InputSimulate.java    License: Apache License 2.0 5 votes vote down vote up
public String getBase64FileKey() {
  String fileKey;
  try {
    fileKey = InetAddress.getLocalHost().getHostAddress() + "|" + getFilePath();
  } catch (Exception e) {
    // skip
    fileKey = "localhost|" + getFilePath();
  }
  return Base64.byteArrayToBase64(fileKey.getBytes());
}
 
Example 7
Source Project: ambari-logsearch   Source File: InputFile.java    License: Apache License 2.0 5 votes vote down vote up
public BufferedReader openLogFile(File logFile) throws Exception {
  BufferedReader br = new BufferedReader(LogsearchReaderFactory.INSTANCE.getReader(logFile));
  fileKey = getFileKeyFromLogFile(logFile);
  base64FileKey = Base64.byteArrayToBase64(fileKey.toString().getBytes());
  logger.info("fileKey=" + fileKey + ", base64=" + base64FileKey + ". " + getShortDescription());
  return br;
}
 
Example 8
private static boolean wasFileRenamed(File folder, String searchFileBase64) {
  for (File file : folder.listFiles()) {
    Object fileKeyObj = FileUtil.getFileKey(file);
    String fileBase64 = Base64.byteArrayToBase64(fileKeyObj.toString().getBytes());
    if (searchFileBase64.equals(fileBase64)) {
      // even though the file name in the checkpoint file is different from the one it was renamed to, checkpoint files are
      // identified by their name, which is generated from the file key, which would be the same for the renamed file
      logger.info("CheckPoint clean: File key matches file " + file.getAbsolutePath() + ", it must have been renamed");
      return true;
    }
  }
  return false;
}
 
Example 9
Source Project: lucene-solr   Source File: VersionedData.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void writeMap(EntryWriter ew) throws IOException {
  ew.put("version", version);
  if (owner != null) {
    ew.put("owner", owner);
  }
  ew.put("mode", mode.toString());
  if (data != null) {
    ew.put("data", Base64.byteArrayToBase64(data));
  }
}
 
Example 10
Source Project: lucene-solr   Source File: Http2SolrClient.java    License: Apache License 2.0 5 votes vote down vote up
private void setBasicAuthHeader(@SuppressWarnings({"rawtypes"})SolrRequest solrRequest, Request req) {
  if (solrRequest.getBasicAuthUser() != null && solrRequest.getBasicAuthPassword() != null) {
    String userPass = solrRequest.getBasicAuthUser() + ":" + solrRequest.getBasicAuthPassword();
    String encoded = Base64.byteArrayToBase64(userPass.getBytes(FALLBACK_CHARSET));
    req.header("Authorization", "Basic " + encoded);
  }
}
 
Example 11
Source Project: lucene-solr   Source File: HttpSolrClient.java    License: Apache License 2.0 5 votes vote down vote up
private void setBasicAuthHeader(@SuppressWarnings({"rawtypes"})SolrRequest request, HttpRequestBase method) throws UnsupportedEncodingException {
  if (request.getBasicAuthUser() != null && request.getBasicAuthPassword() != null) {
    String userPass = request.getBasicAuthUser() + ":" + request.getBasicAuthPassword();
    String encoded = Base64.byteArrayToBase64(userPass.getBytes(FALLBACK_CHARSET));
    method.setHeader(new BasicHeader("Authorization", "Basic " + encoded));
  }
}
 
Example 12
Source Project: lucene-solr   Source File: JWTAuthPlugin.java    License: Apache License 2.0 5 votes vote down vote up
protected String generateAuthDataHeader() {
  JWTIssuerConfig primaryIssuer = getPrimaryIssuer();
  Map<String,Object> data = new HashMap<>();
  data.put(JWTIssuerConfig.PARAM_AUTHORIZATION_ENDPOINT, primaryIssuer.getAuthorizationEndpoint());
  data.put("client_id", primaryIssuer.getClientId());
  data.put("scope", adminUiScope);
  data.put("redirect_uris", redirectUris);
  String headerJson = Utils.toJSONString(data);
  return Base64.byteArrayToBase64(headerJson.getBytes(StandardCharsets.UTF_8));
}
 
Example 13
Source Project: lucene-solr   Source File: PKIAuthenticationPlugin.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressForbidden(reason = "Needs currentTimeMillis to set current time in header")
private Optional<String> generateToken() {
  SolrRequestInfo reqInfo = getRequestInfo();
  String usr;
  if (reqInfo != null) {
    Principal principal = reqInfo.getUserPrincipal();
    if (principal == null) {
      log.debug("generateToken: principal is null");
      //this had a request but not authenticated
      //so we don't not need to set a principal
      return Optional.empty();
    } else {
      usr = principal.getName();
    }
  } else {
    if (!isSolrThread()) {
      //if this is not running inside a Solr threadpool (as in testcases)
      // then no need to add any header
      log.debug("generateToken: not a solr (server) thread");
      return Optional.empty();
    }
    //this request seems to be originated from Solr itself
    usr = "$"; //special name to denote the user is the node itself
  }

  String s = usr + " " + System.currentTimeMillis();

  byte[] payload = s.getBytes(UTF_8);
  byte[] payloadCipher = publicKeyHandler.keyPair.encrypt(ByteBuffer.wrap(payload));
  String base64Cipher = Base64.byteArrayToBase64(payloadCipher);
  log.trace("generateToken: usr={} token={}", usr, base64Cipher);
  return Optional.of(base64Cipher);
}
 
Example 14
Source Project: lucene-solr   Source File: CursorMark.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Generates a Base64 encoded serialized representation of the sort values 
 * encapsulated by this object, for use in cursor requests.
 *
 * @see #parseSerializedTotem
 */
public String getSerializedTotem() {
  if (null == this.values) {
    return CURSOR_MARK_START;
  }

  final List<SchemaField> schemaFields = sortSpec.getSchemaFields();
  final ArrayList<Object> marshalledValues = new ArrayList<>(values.size()+1);
  for (int i = 0; i < schemaFields.size(); i++) {
    SchemaField fld = schemaFields.get(i);
    Object safeValue = values.get(i);
    if (null != fld) {
      FieldType type = fld.getType();
      safeValue = type.marshalSortValue(safeValue);
    }
    marshalledValues.add(safeValue);
  }

  // TODO: we could also encode info about the SortSpec for error checking:
  // the type/name/dir from the SortFields (or a hashCode to act as a checksum) 
  // could help provide more validation beyond just the number of clauses.

  try (JavaBinCodec jbc = new JavaBinCodec(); ByteArrayOutputStream out = new ByteArrayOutputStream(256)) {
    jbc.marshal(marshalledValues, out);
    byte[] rawData = out.toByteArray();
    return Base64.byteArrayToBase64(rawData, 0, rawData.length);
  } catch (Exception ex) {
    throw new SolrException(ErrorCode.SERVER_ERROR,
                            "Unable to format search after totem", ex);
  }
}
 
Example 15
Source Project: lucene-solr   Source File: CryptoKeys.java    License: Apache License 2.0 5 votes vote down vote up
public static PublicKey deserializeX509PublicKey(String pubKey) {
  try {
    KeyFactory keyFactory = KeyFactory.getInstance("RSA");
    X509EncodedKeySpec publicKeySpec = new X509EncodedKeySpec(Base64.base64ToByteArray(pubKey));
    return keyFactory.generatePublic(publicKeySpec);
  } catch (Exception e) {
    throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,e);
  }
}
 
Example 16
Source Project: lucene-solr   Source File: CryptoKeys.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create an RSA key pair with newly generated keys.
 */
public RSAKeyPair() {
  KeyPairGenerator keyGen;
  try {
    keyGen = KeyPairGenerator.getInstance("RSA");
  } catch (NoSuchAlgorithmException e) {
    throw new AssertionError("JVM spec is required to support RSA", e);
  }
  keyGen.initialize(DEFAULT_KEYPAIR_LENGTH);
  java.security.KeyPair keyPair = keyGen.genKeyPair();
  privateKey = keyPair.getPrivate();
  publicKey = keyPair.getPublic();
  pubKeyStr = Base64.byteArrayToBase64(publicKey.getEncoded());
}
 
Example 17
Source Project: lucene-solr   Source File: FieldType.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Marshals a binary field value.
 */
protected static Object marshalBase64SortValue(Object value) {
  if (null == value) {
    return null;
  }
  final BytesRef val = (BytesRef)value;
  return Base64.byteArrayToBase64(val.bytes, val.offset, val.length);
}
 
Example 18
Source Project: lucene-solr   Source File: FieldType.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Unmarshals a binary field value.
 */
protected static Object unmarshalBase64SortValue(Object value) {
  if (null == value) {
    return null;
  }
  final String val = (String)value;
  final byte[] bytes = Base64.base64ToByteArray(val);
  return new BytesRef(bytes);
}
 
Example 19
Source Project: lucene-solr   Source File: BinaryField.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public IndexableField createField(SchemaField field, Object val) {
  if (val == null) return null;
  if (!field.stored()) {
    log.trace("Ignoring unstored binary field: {}", field);
    return null;
  }
  byte[] buf = null;
  int offset = 0, len = 0;
  if (val instanceof byte[]) {
    buf = (byte[]) val;
    len = buf.length;
  } else if (val instanceof ByteBuffer && ((ByteBuffer)val).hasArray()) {
    ByteBuffer byteBuf = (ByteBuffer) val;
    buf = byteBuf.array();
    offset = byteBuf.position();
    len = byteBuf.limit() - byteBuf.position();
  } else {
    String strVal = val.toString();
    //the string has to be a base64 encoded string
    buf = Base64.base64ToByteArray(strVal);
    offset = 0;
    len = buf.length;
  }

  return new org.apache.lucene.document.StoredField(field.getName(), buf, offset, len);
}
 
Example 20
Source Project: lucene-solr   Source File: BinaryField.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Object toNativeType(Object val) {
  if (val instanceof byte[]) {
    return ByteBuffer.wrap((byte[]) val);
  } else if (val instanceof CharSequence) {
    final CharSequence valAsCharSequence = (CharSequence) val;
    return ByteBuffer.wrap(Base64.base64ToByteArray(valAsCharSequence.toString()));
  }
  return super.toNativeType(val);
}
 
Example 21
Source Project: lucene-solr   Source File: JWTAuthPluginIntegrationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void infoRequestValidateXSolrAuthHeaders() throws IOException {
  Map<String, String> headers = getHeaders(baseUrl + "/admin/info/system", null);
  assertEquals("Should have received 401 code", "401", headers.get("code"));
  assertEquals("Bearer realm=\"my-solr-jwt\"", headers.get("WWW-Authenticate"));
  String authData = new String(Base64.base64ToByteArray(headers.get("X-Solr-AuthData")), UTF_8);
  assertEquals("{\n" +
      "  \"scope\":\"solr:admin\",\n" +
      "  \"redirect_uris\":[],\n" +
      "  \"authorizationEndpoint\":\"http://acmepaymentscorp/oauth/auz/authorize\",\n" +
      "  \"client_id\":\"solr-cluster\"}", authData);
}
 
Example 22
Source Project: lucene-solr   Source File: JWTAuthPluginTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void xSolrAuthDataHeader() {
  testConfig.put("adminUiScope", "solr:admin");
  testConfig.put("authorizationEndpoint", "http://acmepaymentscorp/oauth/auz/authorize");
  testConfig.put("clientId", "solr-cluster");
  plugin.init(testConfig);
  String headerBase64 = plugin.generateAuthDataHeader();
  String headerJson = new String(Base64.base64ToByteArray(headerBase64), StandardCharsets.UTF_8);
  Map<String,String> parsed = (Map<String, String>) Utils.fromJSONString(headerJson);
  assertEquals("solr:admin", parsed.get("scope"));
  assertEquals("http://acmepaymentscorp/oauth/auz/authorize", parsed.get("authorizationEndpoint"));
  assertEquals("solr-cluster", parsed.get("client_id"));
}
 
Example 23
@Override
    public IndexableField createField(SchemaField field, Object val /*, float boost*/) {
        if (val == null) return null;
        if (!field.stored()) {
            return null;
        }
        byte[] buf = null;
        int offset = 0, len = 0;
        if (val instanceof byte[]) {
            buf = (byte[]) val;
            len = buf.length;
        } else if (val instanceof ByteBuffer && ((ByteBuffer)val).hasArray()) {
            ByteBuffer byteBuf = (ByteBuffer) val;
            buf = byteBuf.array();
            offset = byteBuf.position();
            len = byteBuf.limit() - byteBuf.position();
        } else {
            String strVal = val.toString();
            //the string has to be a base64 encoded string
            buf = Base64.base64ToByteArray(strVal);
            offset = 0;
            len = buf.length;
        }

        Field f = new org.apache.lucene.document.BinaryDocValuesField(field.getName(), new BytesRef(buf, offset, len));
//        Field f = new org.apache.lucene.document.StoredField(field.getName(), buf, offset, len);
        //f.setBoost(boost);
        return f;
    }
 
Example 24
private static boolean checkCheckPointFile(File checkPointFile) {
  boolean deleted = false;
  try (RandomAccessFile checkPointReader = new RandomAccessFile(checkPointFile, "r")) {
    int contentSize = checkPointReader.readInt();
    byte b[] = new byte[contentSize];
    int readSize = checkPointReader.read(b, 0, contentSize);
    if (readSize != contentSize) {
      logger.error("Couldn't read expected number of bytes from checkpoint file. expected=" + contentSize + ", read="
        + readSize + ", checkPointFile=" + checkPointFile);
    } else {
      String jsonCheckPointStr = new String(b, 0, readSize);
      Map<String, Object> jsonCheckPoint = LogFeederUtil.toJSONObject(jsonCheckPointStr);

      String logFilePath = (String) jsonCheckPoint.get("file_path");
      String logFileKey = (String) jsonCheckPoint.get("file_key");
      Integer maxAgeMin = null;
      if (jsonCheckPoint.containsKey("max_age_min")) {
        maxAgeMin = Integer.parseInt(jsonCheckPoint.get("max_age_min").toString());
      }
      if (logFilePath != null && logFileKey != null) {
        boolean deleteCheckPointFile = false;
        File logFile = new File(logFilePath);
        if (logFile.exists()) {
          Object fileKeyObj = FileUtil.getFileKey(logFile);
          String fileBase64 = Base64.byteArrayToBase64(fileKeyObj.toString().getBytes());
          if (!logFileKey.equals(fileBase64)) {
            logger.info("CheckPoint clean: File key has changed. old=" + logFileKey + ", new=" + fileBase64 + ", filePath=" +
              logFilePath + ", checkPointFile=" + checkPointFile.getAbsolutePath());
            deleteCheckPointFile = !wasFileRenamed(logFile.getParentFile(), logFileKey);
          } else if (maxAgeMin != null && maxAgeMin != 0 && FileUtil.isFileTooOld(logFile, maxAgeMin)) {
            deleteCheckPointFile = true;
            logger.info("Checkpoint clean: File reached max age minutes (" + maxAgeMin + "):" + logFilePath);
          }
        } else {
          logger.info("CheckPoint clean: Log file doesn't exist. filePath=" + logFilePath + ", checkPointFile=" +
            checkPointFile.getAbsolutePath());
          deleteCheckPointFile = !wasFileRenamed(logFile.getParentFile(), logFileKey);
        }
        if (deleteCheckPointFile) {
          logger.info("Deleting CheckPoint file=" + checkPointFile.getAbsolutePath() + ", logFile=" + logFilePath);
          checkPointFile.delete();
          deleted = true;
        }
      }
    }
  } catch (EOFException eof) {
    logger.warn("Caught EOFException. Ignoring reading existing checkPoint file. " + checkPointFile);
  } catch (Throwable t) {
    logger.error("Error while checking checkPoint file. " + checkPointFile, t);
  }

  return deleted;
}
 
Example 25
Source Project: scipio-erp   Source File: ScipioHttpSolrClient.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Sets basic auth header to the given username and password.
 * <p>
 * DEV NOTE: Derived from <code>HttpSolrClient#setBasicAuthHeader</code>, which is private in superclass.
 */
protected void setBasicAuthHeader(HttpRequestBase method, String username, String password) throws UnsupportedEncodingException {
    String userPass = username + ":" + password;
    String encoded = Base64.byteArrayToBase64(userPass.getBytes(StandardCharsets.UTF_8));
    method.setHeader(new BasicHeader("Authorization", "Basic " + encoded));
}
 
Example 26
Source Project: lucene-solr   Source File: SolrCloudAuthTestCase.java    License: Apache License 2.0 4 votes vote down vote up
protected static String makeBasicAuthHeader(String user, String pwd) {
  String userPass = user + ":" + pwd;
  return "Basic " + Base64.byteArrayToBase64(userPass.getBytes(UTF_8));
}
 
Example 27
Source Project: lucene-solr   Source File: MetricsHistoryHandler.java    License: Apache License 2.0 4 votes vote down vote up
private NamedList<Object> getDbData(RrdDb db, String[] dsNames, Format format, SolrParams params) throws IOException {
  NamedList<Object> res = new SimpleOrderedMap<>();
  if (dsNames == null || dsNames.length == 0) {
    dsNames = db.getDsNames();
  }
  StringBuilder str = new StringBuilder();
  RrdDef def = db.getRrdDef();
  ArcDef[] arcDefs = def.getArcDefs();
  for (ArcDef arcDef : arcDefs) {
    SimpleOrderedMap<Object> map = new SimpleOrderedMap<>();
    res.add(arcDef.dump(), map);
    Archive a = db.getArchive(arcDef.getConsolFun(), arcDef.getSteps());
    // startTime / endTime, arcStep are in seconds
    FetchRequest fr = db.createFetchRequest(arcDef.getConsolFun(),
        a.getStartTime() - a.getArcStep(),
        a.getEndTime() + a.getArcStep());
    FetchData fd = fr.fetchData();
    if (format != Format.GRAPH) {
      // add timestamps separately from values
      long[] timestamps = fd.getTimestamps();
      if (format == Format.LIST) {
        // Arrays.asList works only on arrays of Objects
        map.add("timestamps", Arrays.stream(timestamps).boxed().collect(Collectors.toList()));
      } else {
        str.setLength(0);
        for (int i = 0; i < timestamps.length; i++) {
          if (i > 0) {
            str.append('\n');
          }
          str.append(String.valueOf(timestamps[i]));
        }
        map.add("timestamps", str.toString());
      }
    }
    SimpleOrderedMap<Object> values = new SimpleOrderedMap<>();
    map.add("values", values);
    for (String name : dsNames) {
      double[] vals = fd.getValues(name);
      switch (format) {
        case GRAPH:
          RrdGraphDef graphDef = new RrdGraphDef();
          graphDef.setTitle(name);
          graphDef.datasource(name, fd);
          graphDef.setStartTime(a.getStartTime() - a.getArcStep());
          graphDef.setEndTime(a.getEndTime() + a.getArcStep());
          graphDef.setPoolUsed(false);
          graphDef.setAltAutoscale(true);
          graphDef.setAltYGrid(true);
          graphDef.setAltYMrtg(true);
          graphDef.setSignature("Apache Solr " + versionString);
          graphDef.setNoLegend(true);
          graphDef.setAntiAliasing(true);
          graphDef.setTextAntiAliasing(true);
          graphDef.setWidth(500);
          graphDef.setHeight(175);
          graphDef.setTimeZone(TimeZone.getDefault());
          graphDef.setLocale(Locale.ROOT);
          // redraw immediately
          graphDef.setLazy(false);
          // area with a border
          graphDef.area(name, new Color(0xffb860), null);
          graphDef.line(name, Color.RED, null, 1.0f);
          RrdGraph graph = new RrdGraph(graphDef);
          BufferedImage bi = new BufferedImage(
              graph.getRrdGraphInfo().getWidth(),
              graph.getRrdGraphInfo().getHeight(),
              BufferedImage.TYPE_INT_RGB);
          graph.render(bi.getGraphics());
          ByteArrayOutputStream baos = new ByteArrayOutputStream();
          ImageIO.write(bi, "png", baos);
          values.add(name, Base64.byteArrayToBase64(baos.toByteArray()));
          break;
        case STRING:
          str.setLength(0);
          for (int i = 0; i < vals.length; i++) {
            if (i > 0) {
              str.append('\n');
            }
            str.append(String.valueOf(vals[i]));
          }
          values.add(name, str.toString());
          break;
        case LIST:
          values.add(name, Arrays.stream(vals).boxed().collect(Collectors.toList()));
          break;
      }
    }
  }
  return res;
}
 
Example 28
Source Project: lucene-solr   Source File: TermVectorComponent.java    License: Apache License 2.0 4 votes vote down vote up
private void mapOneVector(NamedList<Object> docNL, FieldOptions fieldOptions, IndexReader reader, int docID, TermsEnum termsEnum, String field) throws IOException {
  NamedList<Object> fieldNL = new NamedList<>();
  docNL.add(field, fieldNL);

  BytesRef text;
  PostingsEnum dpEnum = null;
  while((text = termsEnum.next()) != null) {
    String term = text.utf8ToString();
    NamedList<Object> termInfo = new NamedList<>();
    fieldNL.add(term, termInfo);
    final int freq = (int) termsEnum.totalTermFreq();
    if (fieldOptions.termFreq == true) {
      termInfo.add("tf", freq);
    }

    int dpEnumFlags = 0;
    dpEnumFlags |= fieldOptions.positions ? PostingsEnum.POSITIONS : 0;
    //payloads require offsets
    dpEnumFlags |= (fieldOptions.offsets || fieldOptions.payloads) ? PostingsEnum.OFFSETS : 0;
    dpEnumFlags |= fieldOptions.payloads ? PostingsEnum.PAYLOADS : 0;
    dpEnum = termsEnum.postings(dpEnum, dpEnumFlags);

    boolean atNextDoc = false;
    if (dpEnum != null) {
      dpEnum.nextDoc();
      atNextDoc = true;
    }

    if (atNextDoc && dpEnumFlags != 0) {
      NamedList<Integer> positionsNL = null;
      NamedList<Number> theOffsets = null;
      NamedList<String> thePayloads = null;

      for (int i = 0; i < freq; i++) {
        final int pos = dpEnum.nextPosition();
        if (fieldOptions.positions && pos >= 0) {
          if (positionsNL == null) {
            positionsNL = new NamedList<>();
            termInfo.add("positions", positionsNL);
          }
          positionsNL.add("position", pos);
        }

        int startOffset = fieldOptions.offsets ? dpEnum.startOffset() : -1;
        if (startOffset >= 0) {
          if (theOffsets == null) {
            theOffsets = new NamedList<>();
            termInfo.add("offsets", theOffsets);
          }
          theOffsets.add("start", dpEnum.startOffset());
          theOffsets.add("end", dpEnum.endOffset());
        }

        BytesRef payload = fieldOptions.payloads ? dpEnum.getPayload() : null;
        if (payload != null) {
          if (thePayloads == null) {
            thePayloads = new NamedList<>();
            termInfo.add("payloads", thePayloads);
          }
          thePayloads.add("payload", Base64.byteArrayToBase64(payload.bytes, payload.offset, payload.length));
        }
      }
    }
    
    int df = 0;
    if (fieldOptions.docFreq || fieldOptions.tfIdf) {
      df = reader.docFreq(new Term(field, text));
    }

    if (fieldOptions.docFreq) {
      termInfo.add("df", df);
    }

    // TODO: this is not TF/IDF by anyone's definition!
    if (fieldOptions.tfIdf) {
      double tfIdfVal = ((double) freq) / df;
      termInfo.add("tf-idf", tfIdfVal);
    }
  }
}
 
Example 29
Source Project: lucene-solr   Source File: BinaryField.java    License: Apache License 2.0 4 votes vote down vote up
private String toBase64String(ByteBuffer buf) {
  return Base64.byteArrayToBase64(buf.array(), buf.position(), buf.limit()-buf.position());
}
 
Example 30
Source Project: lucene-solr   Source File: JsonPreAnalyzedParser.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public String toFormattedString(Field f) throws IOException {
  Map<String,Object> map = new LinkedHashMap<>();
  map.put(VERSION_KEY, VERSION);
  if (f.fieldType().stored()) {
    String stringValue = f.stringValue();
    if (stringValue != null) {
      map.put(STRING_KEY, stringValue);
    }
    BytesRef binaryValue = f.binaryValue();
    if (binaryValue != null) {
      map.put(BINARY_KEY, Base64.byteArrayToBase64(binaryValue.bytes, binaryValue.offset, binaryValue.length));
    }
  }
  TokenStream ts = f.tokenStreamValue();
  if (ts != null) {
    List<Map<String,Object>> tokens = new LinkedList<>();
    while (ts.incrementToken()) {
      Iterator<Class<? extends Attribute>> it = ts.getAttributeClassesIterator();
      String cTerm = null;
      String tTerm = null;
      Map<String,Object> tok = new TreeMap<>();
      while (it.hasNext()) {
        Class<? extends Attribute> cl = it.next();
        Attribute att = ts.getAttribute(cl);
        if (att == null) {
          continue;
        }
        if (cl.isAssignableFrom(CharTermAttribute.class)) {
          CharTermAttribute catt = (CharTermAttribute)att;
          cTerm = new String(catt.buffer(), 0, catt.length());
        } else if (cl.isAssignableFrom(TermToBytesRefAttribute.class)) {
          TermToBytesRefAttribute tatt = (TermToBytesRefAttribute)att;
          tTerm = tatt.getBytesRef().utf8ToString();
        } else {
          if (cl.isAssignableFrom(FlagsAttribute.class)) {
            tok.put(FLAGS_KEY, Integer.toHexString(((FlagsAttribute)att).getFlags()));
          } else if (cl.isAssignableFrom(OffsetAttribute.class)) {
            tok.put(OFFSET_START_KEY, ((OffsetAttribute)att).startOffset());
            tok.put(OFFSET_END_KEY, ((OffsetAttribute)att).endOffset());
          } else if (cl.isAssignableFrom(PayloadAttribute.class)) {
            BytesRef p = ((PayloadAttribute)att).getPayload();
            if (p != null && p.length > 0) {
              tok.put(PAYLOAD_KEY, Base64.byteArrayToBase64(p.bytes, p.offset, p.length));
            }
          } else if (cl.isAssignableFrom(PositionIncrementAttribute.class)) {
            tok.put(POSINCR_KEY, ((PositionIncrementAttribute)att).getPositionIncrement());
          } else if (cl.isAssignableFrom(TypeAttribute.class)) {
            tok.put(TYPE_KEY, ((TypeAttribute)att).type());
          } else {
            tok.put(cl.getName(), att.toString());
          }
        }
      }
      String term = null;
      if (cTerm != null) {
        term = cTerm;
      } else {
        term = tTerm;
      }
      if (term != null && term.length() > 0) {
        tok.put(TOKEN_KEY, term);
      }
      tokens.add(tok);
    }
    map.put(TOKENS_KEY, tokens);
  }
  return JSONUtil.toJSON(map, -1);
}