com.indeed.util.io.Files Java Examples

The following examples show how to use com.indeed.util.io.Files. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MMapGOV4Function.java    From mph-table with Apache License 2.0 6 votes vote down vote up
public static <T> MMapGOV4Function<T> readFrom(final String path) throws IOException, NoSuchFieldException,
        IllegalAccessException, ClassNotFoundException {
    final GOV4Function<T> gov4Function;
    try (final InputStream file = new FileInputStream(Files.buildPath(path, "GOV4Function.bin"));
         final ObjectInput input = new ObjectInputStream(file)) {
        gov4Function = (GOV4Function<T>)(input.readObject());
    }
    final Field widthField = gov4Function.getClass().getDeclaredField("width");
    widthField.setAccessible(true);
    final int width = (Integer) widthField.get(gov4Function);
    final MMapBuffer buffer = new MMapBuffer(new File(Files.buildPath(path, "signatures.bin")),
                                             FileChannel.MapMode.READ_ONLY, ByteOrder.LITTLE_ENDIAN);
    final MMapLongBigList signaturesData = new MMapLongBigList(buffer.memory().longArray(0, buffer.memory().length() / 8), width);
    final Field dataField = gov4Function.getClass().getDeclaredField("data");
    dataField.setAccessible(true);
    dataField.set(gov4Function, signaturesData);
    return new MMapGOV4Function<>(gov4Function, buffer, width);
}
 
Example #2
Source File: MMapGOV4Function.java    From mph-table with Apache License 2.0 6 votes vote down vote up
public static <T> void writeTo(final GOV4Function<T> gov4Function, final String path) throws NoSuchFieldException,
        IllegalAccessException, IOException {
    final Field dataField = gov4Function.getClass().getDeclaredField("data");
    dataField.setAccessible(true);
    final AbstractBitVector.LongBigListView signaturesData = (AbstractBitVector.LongBigListView) dataField.get(gov4Function);
    final Field bitVectorField = signaturesData.getClass().getDeclaredField("bitVector");
    bitVectorField.setAccessible(true);
    final BitVector bitVector = (BitVector)bitVectorField.get(signaturesData);
    try (final LittleEndianDataOutputStream outputStream = new LittleEndianDataOutputStream(
            new BufferedOutputStream(new FileOutputStream(Files.buildPath(path, "signatures.bin"))))) {
        for (final long value : bitVector.bits()) {
            outputStream.writeLong(value);
        }
    }
    dataField.set(gov4Function, null);
    try (final OutputStream outputStream = new FileOutputStream(Files.buildPath(path, "GOV4Function.bin"));
         final ObjectOutput objectOutput = new ObjectOutputStream(outputStream)) {
        objectOutput.writeObject(gov4Function);
    }
}
 
Example #3
Source File: FileHostsReloader.java    From imhotep with Apache License 2.0 6 votes vote down vote up
private boolean updateHosts() {
    try {
        final List<Host> newHosts = new ArrayList<Host>();
        for (String line : Files.readTextFile(hostsFile)) {
            if (line.startsWith("#")) continue;

            final String[] splitLine = line.split(":", 2);
            if (splitLine.length != 2) {
                log.error("invalid host: "+line);
                continue;
            }
            newHosts.add(new Host(splitLine[0], Integer.parseInt(splitLine[1])));
        }
        Collections.sort(newHosts);
        log.info("reloaded hosts file, new list of hosts: "+newHosts);
        if (!newHosts.equals(hosts)) {
            hosts = newHosts;
        }
        return true;
    } catch (Exception e) {
        log.error("error loading hosts file", e);
        return false;
    }
}
 
Example #4
Source File: TestSquallArchive.java    From imhotep with Apache License 2.0 6 votes vote down vote up
private static void checkDirectory(FileSystem fs, Path tempDir, File localTempDir, File localArchiveDir) throws IOException {
    Files.delete(localArchiveDir.getAbsolutePath());
    final SquallArchiveReader reader = new SquallArchiveReader(fs, tempDir);
    reader.copyToLocal("tmp/tempfile", localTempDir.getAbsolutePath());
    assertTrue(ByteStreams.equal(
            com.google.common.io.Files.newInputStreamSupplier(new File(localArchiveDir, "tempfile")),
            ByteStreams.newInputStreamSupplier(new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
    ));

    Files.delete(localArchiveDir.getAbsolutePath());
    reader.copyAllToLocal(localTempDir.getAbsolutePath());
    assertTrue(ByteStreams.equal(
            com.google.common.io.Files.newInputStreamSupplier(new File(localArchiveDir, "tempfile")),
            ByteStreams.newInputStreamSupplier(new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
    ));
}
 
Example #5
Source File: TestSquallArchive.java    From imhotep with Apache License 2.0 6 votes vote down vote up
private static void readAllTheFiles(FileSystem fs, Path tempDir, File localTempDir2, List<File> tempFiles) throws IOException {
    final SquallArchiveReader reader = new SquallArchiveReader(fs, tempDir);
    final List<FileMetadata> metadata = reader.readMetadata();
    assertEquals(metadata.size(), tempFiles.size());
    for (int i = 0; i < metadata.size(); ++i) {
        final FileMetadata file = metadata.get(i);
        final File tf = tempFiles.get(i);
        assertEquals(tf.getName(), file.getFilename());
        reader.copyToLocal(file, localTempDir2);
        assertTrue(com.google.common.io.Files.equal(tf, new File(localTempDir2, file.getFilename())));
    }

    for (final File f : localTempDir2.listFiles()) {
        if (!f.delete()) throw new IOException();
    }

    reader.copyToLocal("tempfile3", localTempDir2.getAbsolutePath());
    assertTrue(com.google.common.io.Files.equal(new File(localTempDir2, "tempfile3"), findTempFile3(tempFiles)));
}
 
Example #6
Source File: SimpleFlamdexTest.java    From imhotep with Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyFields() throws IOException {
    final String dir = Files.getTempDirectory("flamdex-test", "foo");
    try {
        SimpleFlamdexWriter w = new SimpleFlamdexWriter(dir, 5L, true);
        w.getIntFieldWriter("if1").close();
        w.getStringFieldWriter("sf1").close();
        w.close();

        SimpleFlamdexReader r = SimpleFlamdexReader.open(dir);
        IntTermIterator it = r.getIntTermIterator("if1");
        assertFalse(it.next());
        StringTermIterator sit = r.getStringTermIterator("sf1");
        assertFalse(sit.next());
        it.close();
        sit.close();
        r.close();
    } finally {
        Files.delete(dir);
    }
}
 
Example #7
Source File: TestImhotepClient.java    From imhotep with Apache License 2.0 6 votes vote down vote up
@Override
protected void setUp() throws Exception {
    tempDir1 = Files.getTempDirectory("imhotep", "test");
    tempOptDir1 = Files.getTempDirectory("imhotep", "optimize.test");
    String datasetDir = new File(tempDir1, DATASET).getAbsolutePath();
    new File(datasetDir).mkdir();
    new File(datasetDir, SHARD0).mkdir();
    new File(datasetDir, SHARD1).mkdir();

    tempDir2 = Files.getTempDirectory("imhotep", "test");
    tempOptDir2 = Files.getTempDirectory("imhotep", "optimize.test");
    String datasetDir2 = new File(tempDir2, DATASET).getAbsolutePath();
    new File(datasetDir2).mkdir();
    new File(datasetDir2, SHARD1).mkdir();

    daemon1 = new ImhotepDaemonRunner(tempDir1, tempOptDir1, getFreePort());
    daemon2 = new ImhotepDaemonRunner(tempDir2, tempOptDir2, getFreePort());
}
 
Example #8
Source File: TestZooKeeperConnection.java    From util with Apache License 2.0 5 votes vote down vote up
@After
public void stopZK() {
    if (zk != null) {
        zk.shutdown();
    }
    Files.delete(tempDir);
}
 
Example #9
Source File: SimpleFlamdexTest.java    From imhotep with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetMetric() throws IOException, FlamdexOutOfMemoryException {
    final String dir = Files.getTempDirectory("flamdex-test", "foo");
    try {
        internalTestGetMetric(dir);
    } finally {
        Files.delete(dir);
    }
}
 
Example #10
Source File: SimpleFlamdexTest.java    From imhotep with Apache License 2.0 5 votes vote down vote up
@Test
public void testIt() throws IOException {
    final String dir = Files.getTempDirectory("flamdex-test", "foo");
    try {
        writeAndRead(dir);
    } finally {
        Files.delete(dir);
    }
}
 
Example #11
Source File: TestImhotepClient.java    From imhotep with Apache License 2.0 5 votes vote down vote up
@Override
protected void tearDown() throws Exception {
    if (daemon1 != null) {
        daemon1.stop();
    }
    if (daemon2 != null) {
        daemon2.stop();
    }
    Files.delete(tempDir1);
    Files.delete(tempDir2);
}
 
Example #12
Source File: SimpleFlamdexReader.java    From imhotep with Apache License 2.0 5 votes vote down vote up
protected static void buildStringBTrees(final String directory, final List<String> stringFields) throws IOException {
    for (final String stringField : stringFields) {
        final File btreeDir = new File(Files.buildPath(directory, "fld-" + stringField + ".strindex"));
        if (!btreeDir.exists()) {
            SimpleFlamdexWriter.writeStringBTree(directory, stringField, btreeDir);
        }
    }
}
 
Example #13
Source File: SimpleFlamdexReader.java    From imhotep with Apache License 2.0 5 votes vote down vote up
protected static void buildIntBTrees(final String directory, final List<String> intFields) throws IOException {
    for (final String intField : intFields) {
        final File btreeDir = new File(Files.buildPath(directory, "fld-" + intField + ".intindex"));
        final File btreeDir64 = new File(Files.buildPath(directory, "fld-" + intField + ".intindex64"));
        if (!btreeDir.exists() && !btreeDir64.exists()) {
            SimpleFlamdexWriter.writeIntBTree(directory, intField, btreeDir64);
        }
    }
}
 
Example #14
Source File: GenericFlamdexReader.java    From imhotep with Apache License 2.0 5 votes vote down vote up
@Override
public StringTermIterator getStringTermIterator(String field) {
    final String termsFilename = Files.buildPath(directory, factory.getStringTermsFilename(field));
    final String docsFilename = Files.buildPath(directory, factory.getStringDocsFilename(field));
    try {
        return factory.createStringTermIterator(termsFilename, docsFilename);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 
Example #15
Source File: GenericFlamdexReader.java    From imhotep with Apache License 2.0 5 votes vote down vote up
@Override
public IntTermIterator getIntTermIterator(String field) {
    final String termsFilename = Files.buildPath(directory, factory.getIntTermsFilename(field));
    final String docsFilename = Files.buildPath(directory, factory.getIntDocsFilename(field));
    try {
        return factory.createIntTermIterator(termsFilename, docsFilename);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 
Example #16
Source File: TestSquallArchive.java    From imhotep with Apache License 2.0 5 votes vote down vote up
private static void readHalfTheFiles(FileSystem fs, Path tempDir, File localTempDir2, List<File> tempFiles) throws IOException {
    final SquallArchiveReader reader = new SquallArchiveReader(fs, tempDir);
    final List<FileMetadata> metadata = reader.readMetadata();
    assertEquals(metadata.size(), tempFiles.size() / 2);
    for (int i = 0; i < metadata.size(); ++i) {
        final FileMetadata file = metadata.get(i);
        final File tf = tempFiles.get(i);
        assertEquals(tf.getName(), file.getFilename());
        reader.copyToLocal(file, localTempDir2);
        assertTrue(ByteStreams.equal(
                com.google.common.io.Files.newInputStreamSupplier(tf),
                com.google.common.io.Files.newInputStreamSupplier(new File(localTempDir2, file.getFilename()))
        ));
    }
}
 
Example #17
Source File: TestSquallArchive.java    From imhotep with Apache License 2.0 5 votes vote down vote up
private static void doTheTest(FileSystem fs, Path tempDir, File localTempDir, SquallArchiveCompressor compressor) throws IOException {
    final Random rand = new Random();
    final List<File> tempFiles = new ArrayList<File>();
    for (int i = 0; i < 10; ++i) {
        final File tempFile = new File(localTempDir, "tempfile" + i);
        tempFiles.add(tempFile);
        final int len = rand.nextInt(1024) + 1024;
        final OutputStream os = new FileOutputStream(tempFile);
        for (int j = 0; j < len; ++j) {
            os.write(rand.nextInt(256));
        }
        os.close();
    }

    Collections.shuffle(tempFiles);

    long bytesWritten = writeHalfTheFiles(fs, tempDir, tempFiles, compressor);
    if (compressor == NONE) {
        assertEquals(bytesWritten, getArchiveBytesWritten(fs, tempDir));
    }

    final File localTempDir2 = new File(getTempDir());
    try {
        readHalfTheFiles(fs, tempDir, localTempDir2, tempFiles);
    } finally {
        Files.delete(localTempDir2.getAbsolutePath());
    }

    bytesWritten += writeTheOtherHalf(fs, tempDir, tempFiles, compressor);
    if (compressor == NONE) {
        assertEquals(bytesWritten, getArchiveBytesWritten(fs, tempDir));
    }

    if (!localTempDir2.mkdirs()) throw new IOException();
    try {
        readAllTheFiles(fs, tempDir, localTempDir2, tempFiles);
    } finally {
        Files.delete(localTempDir2.getAbsolutePath());
    }
}
 
Example #18
Source File: SmartDictionarySerializer.java    From mph-table with Apache License 2.0 5 votes vote down vote up
private GOV4Function<String> buildMphFunction() throws IOException {
    final File tempFolder = File.createTempFile("smartDictionarySerializer", ".tmp");
    if (!Files.delete(tempFolder.getAbsolutePath())) {
        throw new IOException("Can't delete tempFolder: " + tempFolder);
    }
    if (!tempFolder.mkdir()) {
        throw new IOException("Can't create tempFolder: " + tempFolder);
    }
    return new GOV4Function.Builder<String>()
            .keys(Arrays.asList(words))
            .tempDir(tempFolder)
            .transform(new TableWriter.SerializerTransformationStrategy<>(new SmartStringSerializer()))
            .build();
}
 
Example #19
Source File: SimpleIntFieldWriter.java    From imhotep with Apache License 2.0 4 votes vote down vote up
public static SimpleIntFieldWriter open(String outputDirectory, String field, long numDocs, boolean writeBTreesOnClose) throws FileNotFoundException {
    final OutputStream termsOutput = new BufferedOutputStream(new FileOutputStream(Files.buildPath(outputDirectory, getTermsFilename(field))), 65536);
    final OutputStream docsOutput = new BufferedOutputStream(new FileOutputStream(Files.buildPath(outputDirectory, getDocsFilename(field))), 65536);
    return new SimpleIntFieldWriter(outputDirectory, field, writeBTreesOnClose, termsOutput, docsOutput, numDocs);
}
 
Example #20
Source File: TestCloseSessionDuringFTGS.java    From imhotep with Apache License 2.0 4 votes vote down vote up
@Test
    public void testCloseSessionDuringFTGS() throws ImhotepOutOfMemoryException, IOException, InterruptedException {
        String tempDir = Files.getTempDirectory("asdf", "");
        try {
            IndexWriter w = new IndexWriter(tempDir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);

            Random rand = new Random();
            for (int i = 0; i < 1000000; ++i) {
                int numTerms = rand.nextInt(5) + 5;
                Document doc = new Document();
                for (int t = 0; t < numTerms; ++t) {
                    doc.add(new Field("sf1", Integer.toString(rand.nextInt(10000)), Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
                }
                w.addDocument(doc);
            }

            w.close();

            final AtomicBoolean closed = new AtomicBoolean(false);
            FlamdexReader r = new LuceneFlamdexReader(IndexReader.open(tempDir)) {
                @Override
                public void close() throws IOException {
                    super.close();
                    closed.set(true);
                }
            };
            final ExecutorService executor = Executors.newCachedThreadPool();
            try {
                ImhotepSession session =
                        new MTImhotepMultiSession(new ImhotepLocalSession[] { new ImhotepLocalSession(r) },
                                                  new MemoryReservationContext(
                                                                               new ImhotepMemoryPool(
                                                                                                     Long.MAX_VALUE)),
                                                  executor, null);
//                FTGSIterator iter = session.getFTGSIterator(new String[]{}, new String[]{"sf1"}); //TODO fix this
                session.close();
                assertTrue(closed.get());
            } finally {
                executor.shutdown();
            }
        } finally {
            Files.delete(tempDir);
        }
    }
 
Example #21
Source File: SimpleStringFieldWriter.java    From imhotep with Apache License 2.0 4 votes vote down vote up
public static SimpleStringFieldWriter open(String outputDirectory, String field, long numDocs, boolean writeBTreesOnClose) throws FileNotFoundException {
    final OutputStream termsOutput = new BufferedOutputStream(new FileOutputStream(Files.buildPath(outputDirectory, getTermsFilename(field))), 65536);
    final OutputStream docsOutput = new BufferedOutputStream(new FileOutputStream(Files.buildPath(outputDirectory, getDocsFilename(field))), 65536);
    return new SimpleStringFieldWriter(outputDirectory, field, writeBTreesOnClose, termsOutput, docsOutput, numDocs);
}
 
Example #22
Source File: TestSimpleFlamdexDocWriter.java    From imhotep with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {
    tempDir = Files.getTempDirectory("flamdex-test", "dir");
}
 
Example #23
Source File: TestSimpleFlamdexDocWriter.java    From imhotep with Apache License 2.0 4 votes vote down vote up
@After
public void tearDown() throws Exception {
    Files.delete(tempDir);
}
 
Example #24
Source File: TestSimpleIterators.java    From imhotep with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {
    tempDir = Files.getTempDirectory("flamdex-test", "dir");
}
 
Example #25
Source File: TestSimpleIterators.java    From imhotep with Apache License 2.0 4 votes vote down vote up
@After
public void tearDown() throws Exception {
    Files.delete(tempDir);
}
 
Example #26
Source File: TestSquallArchive.java    From imhotep with Apache License 2.0 4 votes vote down vote up
private static String getTempDir() {
    return com.google.common.io.Files.createTempDir().getAbsolutePath();
}
 
Example #27
Source File: TestZooKeeperConnection.java    From util with Apache License 2.0 4 votes vote down vote up
@Before
public void startZK() throws IOException {
    tempDir = Files.getTempDirectory("asdf", "");
    zk = new MiniZooKeeperCluster(tempDir);
}